@rocicorp/zero 1.11.0-canary.12 → 1.11.0-canary.13

package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#commitThresholdBytes","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#canceled","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\n/**\n * A {@link MessageBackfill} or {@link BackfillCompleted} paired with the\n * approximate number of upstream bytes it represents. The BackfillManager uses\n * `byteSize` to bound the size of each replica transaction, committing and\n * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.\n */\nexport type BackfillMessage = {\n  message: MessageBackfill | BackfillCompleted;\n  byteSize: number;\n};\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<BackfillMessage>;\n\n/**\n * Commit (and reopen) the current backfill transaction once this many bytes of\n * backfill data have accumulated in it. Bounds the size of a single replica\n * COMMIT/checkpoint so it can't monopolize the replica.\n */\nconst COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  canceledReason?: string | undefined;\n  minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  readonly #commitThresholdBytes: number;\n\n  /** Set when the change stream is canceled. No further backfills are run. */\n  #canceled = false;\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n    commitThresholdBytes = COMMIT_THRESHOLD_BYTES,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n    this.#commitThresholdBytes = commitThresholdBytes;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#canceled &&\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {request, minWatermark: ''};\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    if (this.#canceled) {\n      this.#lc.debug?.(`not retrying backfill: change stream canceled`, e);\n      return;\n    }\n    this.#lc.warn?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n    let uncommittedBytes = 0;\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.canceledReason ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (state.canceledReason === undefined) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      let tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      if (msg.tag === 'backfill-completed' && tx < msg.watermark) {\n        // At this point it must be the case that the #changeStreamReached() the\n        // backfill watermark. Given that guarantee, ensure that the version of the\n        // transaction containing the backfill-completed message is at least up\n        // to the backfill watermark, so that the final database state version is\n        // never earlier than the version of any backfilled rows.\n        tx = msg.watermark;\n      }\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n      }\n      backfillTx = null;\n      uncommittedBytes = 0;\n    };\n\n    for await (const {message: msg, byteSize} of this.#backfillStreamer(\n      state.request,\n    )) {\n      if (this.#canceled) {\n        // Exiting the loop finalizes the backfill stream (and the upstream\n        // resources it holds). The reservation, if held, does not need to be\n        // released since the change stream is gone.\n        lc.info?.(`backfill stream canceled: change stream canceled`);\n        return;\n      }\n      // Before sending `backfill-completed`, the main replication stream\n      // may need to catch up, and/or the current transaction may need to be\n      // committed to open a new transaction that's up to backfill watermark.\n      const mustWaitBeforeFlush =\n        msg.tag === 'backfill-completed' &&\n        (this.#changeStreamReached(lc, msg.watermark) ||\n          (backfillTx !== null && backfillTx < msg.watermark));\n\n      // Commit (and later reopen) the transaction if the main stream is\n      // waiting on the reservation, if we must catch up before completing, or\n      // if the size of current transaction has reached the commit threshold.\n      if (\n        backfillTx &&\n        (changeStream.waiterDelay() > 0 ||\n          mustWaitBeforeFlush ||\n          uncommittedBytes >= this.#commitThresholdBytes)\n      ) {\n        commitTx();\n      }\n\n      mustWaitBeforeFlush && (await mustWaitBeforeFlush);\n\n      if (\n        msg.tag === 'backfill' &&\n        msg.rowValues.length > 0 &&\n        msg.relation.rowKey.columns.length === 0\n      ) {\n        throw new MissingRowKeyError(state.request);\n      }\n\n      // Reserve the changeStreamer if not in a transaction.\n      if ((backfillTx ??= await beginTxFor(msg)) === null) {\n        lc.info?.(\n          `backfill stream canceled: ${state.canceledReason}`,\n          state.request,\n        );\n        this.#checkAndStartBackfill(); // start the next backfill if present\n        return; // this backfill is canceled\n      }\n\n      // `await` to allow the change streamer to exert back pressure\n      // on backfills.\n      await changeStream.push(['data', msg]);\n      uncommittedBytes += byteSize;\n    }\n\n    // Flush any final tx and release the stream.\n    backfillTx && commitTx();\n    lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.canceledReason = reason;\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: backfill,\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: backfill},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: backfill,\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#canceled = true;\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n    this.#backfillRetryTimer = undefined;\n\n    // Wake up a backfill that is waiting for the change stream to reach a\n    // watermark. The change stream never will, and the backfill must be\n    // allowed to unwind (and release its upstream resources) rather than\n    // remain suspended forever.\n    for (const {reached} of this.#awaitingStatusWatermarks.splice(0)) {\n      reached();\n    }\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"mappings":";;;;;;;;;AAyBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;;;;;;AAsBA,IAAM,yBAAyB,IAAI,OAAO;AAQ1C,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC;;CAGA,YAAY;CAEZ,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf,uBAAuB,wBACvB;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKE,kBAAkB;EACvB,KAAKC,oBAAoB;EACzB,KAAKC,cAAc;EACnB,KAAKG,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKH,wBAAwB;CAC/B;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAKN,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAKU,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAKC,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAKC,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAKF,wBAAwB,MAAM,WAAW;GACjD,KAAKA,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAKL,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAKA,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAKA,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAKK,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAKL,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAKK,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAKG,aACN,CAAC,KAAKC,uBACN,CAAC,KAAKC,oBACN,KAAKd,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAKA,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IAAC;IAAS,cAAc;GAAE;GACxC,MAAM,KAAK,KAAKD,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAKe,mBAAmB;GACxB,KAAUC,aAAa,IAAI,KAAK,EAC7B,WAAW;IACV,KAAKC,qBAAqB,mBAAmB,KAAK;IAClD,KAAKR,gBAAgB,KAAKF;GAC5B,CAAC,EAGA,OAAM,MAAK;IACV,KAAKU,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAKC,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,IAAI,KAAKL,WAAW;GAClB,KAAKb,IAAI,QAAQ,iDAAiD,CAAC;GACnE;EACF;EACA,KAAKA,IAAI,OACP,uCAAuC,KAAKS,cAAc,MAC1D,CACF;EACA,KAAKK,sBAAsB,iBAAiB;GAC1C,KAAKA,sBAAsB,KAAA;GAC3B,KAAKF,uBAAuB;EAC9B,GAAG,KAAKH,aAAa;EAErB,KAAKA,gBAAgB,KAAK,IAAI,KAAKA,gBAAgB,GAAG,KAAKD,aAAa;CAC1E;CAEA,MAAMQ,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAKd;EAI1B,IAAI,aAA4B;EAChC,IAAI,mBAAmB;;;;EAKvB,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,MAAM,mBAAmB,KAAA,GAAW;KACtC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAKe,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,IAAI,QAAQ,wBAAwB,KAAK,IAAI,WAM/C,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAKb;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;GACjC;GACA,aAAa;GACb,mBAAmB;EACrB;EAEA,WAAW,MAAM,EAAC,SAAS,KAAK,cAAa,KAAKD,kBAChD,MAAM,OACR,GAAG;GACD,IAAI,KAAKU,WAAW;IAIlB,GAAG,OAAO,kDAAkD;IAC5D;GACF;GAIA,MAAM,sBACJ,IAAI,QAAQ,yBACX,KAAKM,qBAAqB,IAAI,IAAI,SAAS,KACzC,eAAe,QAAQ,aAAa,IAAI;GAK7C,IACE,eACC,aAAa,YAAY,IAAI,KAC5B,uBACA,oBAAoB,KAAKb,wBAE3B,SAAS;GAGX,uBAAwB,MAAM;GAE9B,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;GAI5C,KAAK,eAAe,MAAM,WAAW,GAAG,OAAO,MAAM;IACnD,GAAG,OACD,6BAA6B,MAAM,kBACnC,MAAM,OACR;IACA,KAAKM,uBAAuB;IAC5B;GACF;GAIA,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;GACrC,oBAAoB;EACtB;EAGA,cAAc,SAAS;EACvB,GAAG,QAAQ,0BAA0B,MAAM,kBAAkB,EAAE;CACjE;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAKG;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAKA;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,iBAAiB;GAC1B,KAAKA,mBAAmB;GACxB,UAAU,KAAKf,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAKC,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAKC,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAKA,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAKC,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAKmB,sBAAsB,QAAQ,GAAG;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKA,sBAAsB;GAC3B,KAAKC,wBAAwB,QAAQ,EAAE;GAGvC,KAAKT,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKS,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAKpB,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAKU,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAKW,oBAAoB,KAAK,GAChC,KAAKL,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAKN,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS;IACX,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAKV,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAKsB,wBAAwB,KAAK,GAAG;KACrC,KAAKZ,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAKW,oBAAoB,GAAG,GAC9B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAKhB,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAKsB,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAKD,oBAAoB,EAAE,GAC7B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAKhB,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAKU,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,SAAQ;KACnC,CAAC;UAED,KAAKA,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO;MACjB;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAKV,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAKU,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAKW,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAKL,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAKhB,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,EAAE,WAAW,GACpC,KAAKsB,wBAAwB,KAAK,KAAK;UAEvC,KAAKZ,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAKW,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAKL,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAKK,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAKF,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAKpB,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAKC,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,EAAE,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAKsB,wBAAwB,KAAK,QAAQ;SAE1C,KAAKZ,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAKM,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAKJ,YAAY;EACjB,KAAKI,qBAAqB,wBAAwB;EAClD,aAAa,KAAKH,mBAAmB;EACrC,KAAKA,sBAAsB,KAAA;EAM3B,KAAK,MAAM,EAAC,aAAY,KAAKT,0BAA0B,OAAO,CAAC,GAC7D,QAAQ;CAEZ;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}
+{"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#commitThresholdBytes","#controller","#canceled","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#maybeLogBackfillStats","#logBackfillStats","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\n/**\n * A {@link MessageBackfill} or {@link BackfillCompleted} paired with the\n * approximate number of upstream bytes it represents. The BackfillManager uses\n * `byteSize` to bound the size of each replica transaction, committing and\n * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.\n */\nexport type BackfillMessage = {\n  message: MessageBackfill | BackfillCompleted;\n  byteSize: number;\n};\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<BackfillMessage>;\n\n/**\n * Commit (and reopen) the current backfill transaction once this many bytes of\n * backfill data have accumulated in it. Bounds the size of a single replica\n * COMMIT/checkpoint so it can't monopolize the replica.\n */\nconst COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;\n\n/**\n * How many backfill messages the prefetch producer may read ahead of the\n * reservation-holding consumer loop. This bounds the in-memory buffer (~this\n * many {@link BackfillMessage}s, each up to the stream's flush threshold, e.g.\n * 64 KiB) while giving the consumer a supply of ready messages so it never\n * has to wait on the upstream COPY while holding the change-stream reservation.\n */\nconst BACKFILL_PREFETCH_CREDITS = 16;\n\n/**\n * While the backfill holds an open transaction but its prefetch queue is empty\n * (i.e. the upstream is momentarily behind), the consumer polls for the next\n * message at this interval so that a replication transaction that begins\n * waiting mid-fetch can preempt the backfill within this bound, rather than\n * being blocked for the entire (possibly slow) upstream read.\n *\n * If stats indicate that these stalls are highly correlative with replication\n * lag (indicating that replication changes are waiting on these polls), we\n * can address this by adding an API to the change stream multiplexer that\n * would allow backfill to relinquish an in-progress commit on demand (i.e.\n * only if replication asks for a reservation).\n */\nconst BACKFILL_WAITER_POLL_MS = 10;\n\n/** Interval at which in-progress backfill timing stats are logged. */\nconst BACKFILL_STATS_LOG_INTERVAL_MS = 10_000;\n\n/** Sentinel enqueued by the prefetch producer when the backfill stream ends. */\nconst DONE = Symbol('backfill-done');\n/** Sentinel returned by a polled dequeue that timed out (see {@link BACKFILL_WAITER_POLL_MS}). */\nconst POLL = Symbol('backfill-poll');\n\ntype Pull = BackfillMessage | typeof DONE | typeof POLL;\n\n/**\n * Accumulated timing attribution for a single backfill run, logged\n * periodically to reveal where the backfill spends its time (and, in\n * particular, where the change-stream reservation is held). `produceMs`\n * accrues on the prefetch producer task and thus overlaps with the consumer's\n * `pushMs`; the others accrue on the reservation-holding consumer loop.\n */\ntype BackfillTimingStats = {\n  start: number;\n  lastLogged: number;\n  messages: number;\n  bytes: number;\n  commits: number;\n  stalls: number;\n  produceMs: number;\n  pushMs: number;\n  reserveMs: number;\n  stallMs: number;\n};\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  canceledReason?: string | undefined;\n  minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  readonly #commitThresholdBytes: number;\n\n  /**\n   * Aborted when the change stream is canceled. No further backfills are run.\n   *\n   * TODO: Thread this into the backfillStream so that upstream I/O can be\n   * aborted.\n   */\n  readonly #controller = new AbortController();\n\n  get #canceled() {\n    return this.#controller.signal.aborted;\n  }\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n    commitThresholdBytes = COMMIT_THRESHOLD_BYTES,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n    this.#commitThresholdBytes = commitThresholdBytes;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#canceled &&\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {request, minWatermark: ''};\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    if (this.#canceled) {\n      this.#lc.debug?.(`not retrying backfill: change stream canceled`, e);\n      return;\n    }\n    this.#lc.warn?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n    let uncommittedBytes = 0;\n\n    const stats: BackfillTimingStats = {\n      start: performance.now(),\n      lastLogged: performance.now(),\n      messages: 0,\n      bytes: 0,\n      commits: 0,\n      stalls: 0,\n      produceMs: 0,\n      pushMs: 0,\n      reserveMs: 0,\n      stallMs: 0,\n    };\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.canceledReason ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (state.canceledReason === undefined) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      let tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      if (msg.tag === 'backfill-completed' && tx < msg.watermark) {\n        // At this point it must be the case that the #changeStreamReached() the\n        // backfill watermark. Given that guarantee, ensure that the version of the\n        // transaction containing the backfill-completed message is at least up\n        // to the backfill watermark, so that the final database state version is\n        // never earlier than the version of any backfilled rows.\n        tx = msg.watermark;\n      }\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n        stats.commits++;\n      }\n      backfillTx = null;\n      uncommittedBytes = 0;\n    };\n\n    // === Prefetch producer ===\n    //\n    // The backfill stream (an upstream Postgres COPY) is drained on a separate\n    // task into a bounded queue, so that the reservation-holding consumer loop\n    // below never blocks on the upstream read while it holds the change-stream\n    // reservation. `credits` bounds how far the producer runs ahead; the\n    // consumer returns a credit for every message it takes.\n    const ready = new Queue<Pull>();\n    const credits = new Queue<1>();\n    for (let i = 0; i < BACKFILL_PREFETCH_CREDITS; i++) {\n      credits.enqueue(1);\n    }\n    let stopProducer = false;\n\n    // Wakes a consumer blocked awaiting the next prefetched message so a\n    // canceled backfill unwinds promptly.\n    const onCancel = () =>\n      ready.enqueueRejection(new AbortError('backfill canceled'));\n    if (!this.#canceled) {\n      this.#controller.signal.addEventListener('abort', onCancel);\n    }\n\n    const producer = (async () => {\n      try {\n        let t0 = performance.now();\n        for await (const value of this.#backfillStreamer(state.request)) {\n          stats.produceMs += performance.now() - t0;\n          await credits.dequeue(); // wait for buffer space (flow control)\n          if (stopProducer || this.#canceled) {\n            ready.enqueueRejection(new AbortError('backfill canceled'));\n            return;\n          }\n          ready.enqueue(value);\n          t0 = performance.now();\n        }\n        ready.enqueue(DONE);\n      } catch (e) {\n        ready.enqueueRejection(e);\n      }\n    })();\n    // The producer routes errors through `ready` and never rejects; guard\n    // anyway so that leaving it un-awaited on the cancellation path (below)\n    // cannot surface as an unhandled rejection.\n    void producer.catch(() => {});\n\n    // === Consumer ===\n\n    // Fetches the next item from the prefetch queue. If a transaction is open\n    // but the queue is empty (the producer/upstream is momentarily behind),\n    // polls so that a replication transaction that begins waiting during the\n    // upstream read can preempt the backfill: rather than hold the reservation\n    // for the whole read, commit and release, letting replication proceed.\n    const nextItem = async (): Promise<BackfillMessage | typeof DONE> => {\n      // Fast path: a message is already buffered, or there's no reservation to\n      // protect. Take it without altering transaction boundaries. `POLL` is\n      // never enqueued (only used as a dequeue timeout value), so the result is\n      // always a message or DONE.\n      if (backfillTx === null || ready.size() > 0) {\n        const n = ready.dequeue();\n        const v = n instanceof Promise ? await n : n;\n        return v as BackfillMessage | typeof DONE;\n      }\n      // Slow path: holding a reservation with an empty queue.\n      for (;;) {\n        if (backfillTx !== null && changeStream.waiterDelay() > 0) {\n          commitTx(); // yield the stream to the waiting replication txn\n        }\n        const n =\n          backfillTx !== null\n            ? ready.dequeue(POLL, BACKFILL_WAITER_POLL_MS)\n            : ready.dequeue(); // released: just wait for the next message\n        const t0 = performance.now();\n        const v = n instanceof Promise ? await n : n;\n        if (v === POLL) {\n          stats.stallMs += performance.now() - t0; // timed out; re-check waiter\n          continue;\n        }\n        if (n instanceof Promise) {\n          stats.stallMs += performance.now() - t0;\n          stats.stalls++;\n        }\n        return v;\n      }\n    };\n\n    try {\n      for (;;) {\n        const item = await nextItem();\n        if (item === DONE) {\n          break;\n        }\n        // Allow the producer to fetch the next message concurrently with the\n        // processing (and downstream push) of this one.\n        credits.enqueue(1);\n\n        if (this.#canceled) {\n          // Returning finalizes the backfill stream (and the upstream resources\n          // it holds) via the producer's `finally`. The reservation, if held,\n          // does not need to be released since the change stream is gone.\n          lc.info?.(`backfill stream canceled: change stream canceled`);\n          return;\n        }\n        const {message: msg, byteSize} = item;\n\n        // Before sending `backfill-completed`, the main replication stream\n        // may need to catch up, and/or the current transaction may need to be\n        // committed to open a new transaction that's up to backfill watermark.\n        const mustWaitBeforeFlush =\n          msg.tag === 'backfill-completed' &&\n          (this.#changeStreamReached(lc, msg.watermark) ||\n            (backfillTx !== null && backfillTx < msg.watermark));\n\n        // Commit (and later reopen) the transaction if the main stream is\n        // waiting on the reservation, if we must catch up before completing, or\n        // if the size of current transaction has reached the commit threshold.\n        if (\n          backfillTx &&\n          (changeStream.waiterDelay() > 0 ||\n            mustWaitBeforeFlush ||\n            uncommittedBytes >= this.#commitThresholdBytes)\n        ) {\n          commitTx();\n        }\n\n        if (mustWaitBeforeFlush) {\n          await mustWaitBeforeFlush;\n        }\n\n        if (\n          msg.tag === 'backfill' &&\n          msg.rowValues.length > 0 &&\n          msg.relation.rowKey.columns.length === 0\n        ) {\n          throw new MissingRowKeyError(state.request);\n        }\n\n        // Reserve the changeStreamer if not in a transaction.\n        if (backfillTx === null) {\n          const t0 = performance.now();\n          backfillTx = await beginTxFor(msg);\n          stats.reserveMs += performance.now() - t0;\n          if (backfillTx === null) {\n            lc.info?.(\n              `backfill stream canceled: ${state.canceledReason}`,\n              state.request,\n            );\n            this.#checkAndStartBackfill(); // start the next backfill if present\n            return; // this backfill is canceled\n          }\n        }\n\n        // `await` to allow the change streamer to exert back pressure\n        // on backfills.\n        const t0 = performance.now();\n        await changeStream.push(['data', msg]);\n        stats.pushMs += performance.now() - t0;\n        uncommittedBytes += byteSize;\n        stats.messages++;\n        stats.bytes += byteSize;\n\n        this.#maybeLogBackfillStats(lc, stats);\n      }\n\n      // Flush any final tx and release the stream.\n      if (backfillTx) {\n        commitTx();\n      }\n      lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n    } finally {\n      stopProducer = true;\n      credits.enqueue(1);\n      // The producer can potentially be hung on upstream I/O.\n      // TODO: Thread AbortSignal logic into the backfillStreamer.\n      if (!this.#canceled) {\n        await producer;\n      }\n      this.#controller.signal.removeEventListener('abort', onCancel);\n      this.#logBackfillStats(lc, stats);\n    }\n  }\n\n  #maybeLogBackfillStats(lc: LogContext, stats: BackfillTimingStats) {\n    const now = performance.now();\n    if (now - stats.lastLogged >= BACKFILL_STATS_LOG_INTERVAL_MS) {\n      this.#logBackfillStats(lc, stats);\n      stats.lastLogged = now;\n    }\n  }\n\n  /**\n   * Logs where the backfill spent its time. `produceMs` (upstream read +\n   * parse/decode) accrues on the prefetch producer concurrently with the\n   * consumer's `pushMs` (downstream backpressure), so the two can overlap and\n   * their sum can exceed `elapsedMs`. `reserveMs` is time spent (re)acquiring\n   * the change-stream reservation, and `stallMs` is time the consumer waited on\n   * an empty prefetch queue (i.e. blocked on the upstream) — a large `stallMs`\n   * indicates an upstream-bound backfill.\n   */\n  #logBackfillStats(lc: LogContext, stats: BackfillTimingStats) {\n    const elapsed = performance.now() - stats.start;\n    lc.info?.(`backfill timing`, {\n      messages: stats.messages,\n      bytes: stats.bytes,\n      commits: stats.commits,\n      stalls: stats.stalls,\n      elapsedMs: Math.round(elapsed),\n      produceMs: Math.round(stats.produceMs),\n      pushMs: Math.round(stats.pushMs),\n      reserveMs: Math.round(stats.reserveMs),\n      stallMs: Math.round(stats.stallMs),\n      throughputKBps:\n        elapsed > 0 ? Math.round(stats.bytes / 1024 / (elapsed / 1000)) : 0,\n    });\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.canceledReason = reason;\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: backfill,\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: backfill},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: backfill,\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#controller.abort('canceled');\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n    this.#backfillRetryTimer = undefined;\n\n    // Wake up a backfill that is waiting for the change stream to reach a\n    // watermark. The change stream never will, and the backfill must be\n    // allowed to unwind (and release its upstream resources) rather than\n    // remain suspended forever.\n    for (const {reached} of this.#awaitingStatusWatermarks.splice(0)) {\n      reached();\n    }\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"mappings":";;;;;;;;;;;AA2BA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;;;;;;AAsBA,IAAM,yBAAyB,IAAI,OAAO;;;;;;;;AAS1C,IAAM,4BAA4B;;;;;;;;;;;;;;AAelC,IAAM,0BAA0B;;AAGhC,IAAM,iCAAiC;;AAGvC,IAAM,OAAO,OAAO,eAAe;;AAEnC,IAAM,OAAO,OAAO,eAAe;AA8BnC,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC;;;;;;;CAQA,cAAuB,IAAI,gBAAgB;CAE3C,IAAIQ,YAAY;EACd,OAAO,KAAKD,YAAY,OAAO;CACjC;CAEA,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf,uBAAuB,wBACvB;EACA,KAAKP,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKE,kBAAkB;EACvB,KAAKC,oBAAoB;EACzB,KAAKC,cAAc;EACnB,KAAKK,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKL,wBAAwB;CAC/B;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAKN,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAKY,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAKC,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAKC,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAKF,wBAAwB,MAAM,WAAW;GACjD,KAAKA,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAKP,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAKA,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAKA,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAKO,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAKP,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAKO,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAKJ,aACN,CAAC,KAAKO,uBACN,CAAC,KAAKC,oBACN,KAAKf,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAKA,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IAAC;IAAS,cAAc;GAAE;GACxC,MAAM,KAAK,KAAKD,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAKgB,mBAAmB;GACxB,KAAUC,aAAa,IAAI,KAAK,EAC7B,WAAW;IACV,KAAKC,qBAAqB,mBAAmB,KAAK;IAClD,KAAKP,gBAAgB,KAAKF;GAC5B,CAAC,EAGA,OAAM,MAAK;IACV,KAAKS,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAKC,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,IAAI,KAAKX,WAAW;GAClB,KAAKR,IAAI,QAAQ,iDAAiD,CAAC;GACnE;EACF;EACA,KAAKA,IAAI,OACP,uCAAuC,KAAKW,cAAc,MAC1D,CACF;EACA,KAAKI,sBAAsB,iBAAiB;GAC1C,KAAKA,sBAAsB,KAAA;GAC3B,KAAKD,uBAAuB;EAC9B,GAAG,KAAKH,aAAa;EAErB,KAAKA,gBAAgB,KAAK,IAAI,KAAKA,gBAAgB,GAAG,KAAKD,aAAa;CAC1E;CAEA,MAAMO,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAKf;EAI1B,IAAI,aAA4B;EAChC,IAAI,mBAAmB;EAEvB,MAAM,QAA6B;GACjC,OAAO,YAAY,IAAI;GACvB,YAAY,YAAY,IAAI;GAC5B,UAAU;GACV,OAAO;GACP,SAAS;GACT,QAAQ;GACR,WAAW;GACX,QAAQ;GACR,WAAW;GACX,SAAS;EACX;;;;EAKA,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,MAAM,mBAAmB,KAAA,GAAW;KACtC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAKgB,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,IAAI,QAAQ,wBAAwB,KAAK,IAAI,WAM/C,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAKd;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;IAC/B,MAAM;GACR;GACA,aAAa;GACb,mBAAmB;EACrB;EASA,MAAM,QAAQ,IAAI,MAAY;EAC9B,MAAM,UAAU,IAAI,MAAS;EAC7B,KAAK,IAAI,IAAI,GAAG,IAAI,2BAA2B,KAC7C,QAAQ,QAAQ,CAAC;EAEnB,IAAI,eAAe;EAInB,MAAM,iBACJ,MAAM,iBAAiB,IAAI,WAAW,mBAAmB,CAAC;EAC5D,IAAI,CAAC,KAAKI,WACR,KAAKD,YAAY,OAAO,iBAAiB,SAAS,QAAQ;EAG5D,MAAM,YAAY,YAAY;GAC5B,IAAI;IACF,IAAI,KAAK,YAAY,IAAI;IACzB,WAAW,MAAM,SAAS,KAAKJ,kBAAkB,MAAM,OAAO,GAAG;KAC/D,MAAM,aAAa,YAAY,IAAI,IAAI;KACvC,MAAM,QAAQ,QAAQ;KACtB,IAAI,gBAAgB,KAAKK,WAAW;MAClC,MAAM,iBAAiB,IAAI,WAAW,mBAAmB,CAAC;MAC1D;KACF;KACA,MAAM,QAAQ,KAAK;KACnB,KAAK,YAAY,IAAI;IACvB;IACA,MAAM,QAAQ,IAAI;GACpB,SAAS,GAAG;IACV,MAAM,iBAAiB,CAAC;GAC1B;EACF,GAAG;EAIH,SAAc,YAAY,CAAC,CAAC;EAS5B,MAAM,WAAW,YAAoD;GAKnE,IAAI,eAAe,QAAQ,MAAM,KAAK,IAAI,GAAG;IAC3C,MAAM,IAAI,MAAM,QAAQ;IAExB,OADU,aAAa,UAAU,MAAM,IAAI;GAE7C;GAEA,SAAS;IACP,IAAI,eAAe,QAAQ,aAAa,YAAY,IAAI,GACtD,SAAS;IAEX,MAAM,IACJ,eAAe,OACX,MAAM,QAAQ,MAAM,uBAAuB,IAC3C,MAAM,QAAQ;IACpB,MAAM,KAAK,YAAY,IAAI;IAC3B,MAAM,IAAI,aAAa,UAAU,MAAM,IAAI;IAC3C,IAAI,MAAM,MAAM;KACd,MAAM,WAAW,YAAY,IAAI,IAAI;KACrC;IACF;IACA,IAAI,aAAa,SAAS;KACxB,MAAM,WAAW,YAAY,IAAI,IAAI;KACrC,MAAM;IACR;IACA,OAAO;GACT;EACF;EAEA,IAAI;GACF,SAAS;IACP,MAAM,OAAO,MAAM,SAAS;IAC5B,IAAI,SAAS,MACX;IAIF,QAAQ,QAAQ,CAAC;IAEjB,IAAI,KAAKA,WAAW;KAIlB,GAAG,OAAO,kDAAkD;KAC5D;IACF;IACA,MAAM,EAAC,SAAS,KAAK,aAAY;IAKjC,MAAM,sBACJ,IAAI,QAAQ,yBACX,KAAKY,qBAAqB,IAAI,IAAI,SAAS,KACzC,eAAe,QAAQ,aAAa,IAAI;IAK7C,IACE,eACC,aAAa,YAAY,IAAI,KAC5B,uBACA,oBAAoB,KAAKd,wBAE3B,SAAS;IAGX,IAAI,qBACF,MAAM;IAGR,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;IAI5C,IAAI,eAAe,MAAM;KACvB,MAAM,KAAK,YAAY,IAAI;KAC3B,aAAa,MAAM,WAAW,GAAG;KACjC,MAAM,aAAa,YAAY,IAAI,IAAI;KACvC,IAAI,eAAe,MAAM;MACvB,GAAG,OACD,6BAA6B,MAAM,kBACnC,MAAM,OACR;MACA,KAAKQ,uBAAuB;MAC5B;KACF;IACF;IAIA,MAAM,KAAK,YAAY,IAAI;IAC3B,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;IACrC,MAAM,UAAU,YAAY,IAAI,IAAI;IACpC,oBAAoB;IACpB,MAAM;IACN,MAAM,SAAS;IAEf,KAAKO,uBAAuB,IAAI,KAAK;GACvC;GAGA,IAAI,YACF,SAAS;GAEX,GAAG,QAAQ,0BAA0B,MAAM,kBAAkB,EAAE;EACjE,UAAU;GACR,eAAe;GACf,QAAQ,QAAQ,CAAC;GAGjB,IAAI,CAAC,KAAKb,WACR,MAAM;GAER,KAAKD,YAAY,OAAO,oBAAoB,SAAS,QAAQ;GAC7D,KAAKe,kBAAkB,IAAI,KAAK;EAClC;CACF;CAEA,uBAAuB,IAAgB,OAA4B;EACjE,MAAM,MAAM,YAAY,IAAI;EAC5B,IAAI,MAAM,MAAM,cAAc,gCAAgC;GAC5D,KAAKA,kBAAkB,IAAI,KAAK;GAChC,MAAM,aAAa;EACrB;CACF;;;;;;;;;;CAWA,kBAAkB,IAAgB,OAA4B;EAC5D,MAAM,UAAU,YAAY,IAAI,IAAI,MAAM;EAC1C,GAAG,OAAO,mBAAmB;GAC3B,UAAU,MAAM;GAChB,OAAO,MAAM;GACb,SAAS,MAAM;GACf,QAAQ,MAAM;GACd,WAAW,KAAK,MAAM,OAAO;GAC7B,WAAW,KAAK,MAAM,MAAM,SAAS;GACrC,QAAQ,KAAK,MAAM,MAAM,MAAM;GAC/B,WAAW,KAAK,MAAM,MAAM,SAAS;GACrC,SAAS,KAAK,MAAM,MAAM,OAAO;GACjC,gBACE,UAAU,IAAI,KAAK,MAAM,MAAM,QAAQ,QAAQ,UAAU,IAAK,IAAI;EACtE,CAAC;CACH;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAKN;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAKA;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,iBAAiB;GAC1B,KAAKA,mBAAmB;GACxB,UAAU,KAAKhB,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAKC,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAKC,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAKA,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAKC,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAKsB,sBAAsB,QAAQ,GAAG;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKA,sBAAsB;GAC3B,KAAKC,wBAAwB,QAAQ,EAAE;GAGvC,KAAKV,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKU,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAKvB,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAKY,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAKY,oBAAoB,KAAK,GAChC,KAAKP,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAKL,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS;IACX,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAKZ,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAKyB,wBAAwB,KAAK,GAAG;KACrC,KAAKb,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAKY,oBAAoB,GAAG,GAC9B,KAAKP,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAKjB,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAKyB,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAKD,oBAAoB,EAAE,GAC7B,KAAKP,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAKjB,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAKY,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,SAAQ;KACnC,CAAC;UAED,KAAKA,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO;MACjB;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAKZ,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAKY,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAKY,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAKP,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAKjB,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,EAAE,WAAW,GACpC,KAAKyB,wBAAwB,KAAK,KAAK;UAEvC,KAAKb,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAKY,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAKP,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAKO,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAKF,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAKvB,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAKC,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,EAAE,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAKyB,wBAAwB,KAAK,QAAQ;SAE1C,KAAKb,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAKK,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAKX,YAAY,MAAM,UAAU;EACjC,KAAKW,qBAAqB,wBAAwB;EAClD,aAAa,KAAKH,mBAAmB;EACrC,KAAKA,sBAAsB,KAAA;EAM3B,KAAK,MAAM,EAAC,aAAY,KAAKV,0BAA0B,OAAO,CAAC,GAC7D,QAAQ;CAEZ;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}

package/out/zero-cache/src/services/change-source/custom/change-source.js

@@ -13,7 +13,7 @@ import { ChangeProcessor } from "../../replicator/change-processor.js";
 import { ReplicationStatusPublisher } from "../../replicator/replication-status.js";
 import { restoreReplica } from "../common/replica-restore.js";
 import { initReplica } from "../common/replica-schema.js";
-import { WebSocket } from "ws";
+import { WebSocket as WebSocket$1 } from "ws";
 //#region ../zero-cache/src/services/change-source/custom/change-source.ts
 /**
 * Initializes a Custom change source before streaming changes from the
@@ -78,7 +78,7 @@ var CustomChangeSource = class {
 			url.searchParams.set("lastWatermark", clientWatermark);
 			url.searchParams.set("replicaVersion", replicaVersion);
 		}
-		const ws = new WebSocket(url);
+		const ws = new WebSocket$1(url);
 		const { instream, outstream } = stream(this.#lc, ws, changeStreamMessageSchema, { coalesce: (curr) => curr });
 		return {
 			changes: instream,

package/out/zero-cache/src/services/change-source/custom/change-source.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n  stringify,\n  type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n  AutoResetSignal,\n  type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n  createReplicationStateTables,\n  getSubscriptionState,\n  initReplicationState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {\n  restoreReplica,\n  type InitializeResult,\n  type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n  type BackfillRequest,\n  type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  context: ServerContext,\n  {litestream, constraints}: RestoreOptions = {},\n): Promise<InitializeResult> {\n  // At the moment, the custom change-source implementation does not support\n  // per-RM backups (and thus does not support litestream v5 or RMv2). The\n  // current proposal to support this without requiring additional upstream\n  // state tracking is to:\n  // - use litestream to (ltx) query a fixed pool of subfolders within the\n  //   backupURL (e.g. /a/, /b/, /c/, /d/, /e/)\n  // - restore from the subfolder with the latest ltx file\n  // - backup to the subfolder with the oldest (or non-existent) ltx file\n  if (litestream?.backupURL) {\n    await restoreReplica(lc, litestream, replicaDbFile, constraints);\n  }\n\n  let initialSynced = false;\n  await initReplica(\n    lc,\n    `replica-${shard.appID}-${shard.shardNum}`,\n    replicaDbFile,\n    (log, tx) => {\n      initialSynced = true;\n      return initialSync(log, shard, tx, upstreamURI, context);\n    },\n  );\n\n  const replica = new Database(lc, replicaDbFile);\n  const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n  replica.close();\n\n  if (shard.publications.length) {\n    // Verify that the publications match what has been synced.\n    const requested = shard.publications.toSorted();\n    const replicated = subscriptionState.publications.sort();\n    if (!deepEqual(requested, replicated)) {\n      throw new Error(\n        `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n      );\n    }\n  }\n\n  const changeSource = new CustomChangeSource(\n    lc,\n    upstreamURI,\n    shard,\n    subscriptionState,\n  );\n\n  return {\n    subscriptionState,\n    changeSource,\n    destinationBackupURL: litestream?.backupURL,\n    // A custom change source has no upstream `replicas` table to identify\n    // itself from, so the generation is the whole of the SQLite change log's\n    // identity here.\n    replicaID: null,\n    waitForBackupBeforeServing: initialSynced,\n  };\n}\n\nclass CustomChangeSource implements ChangeSource {\n  readonly #lc: LogContext;\n  readonly #upstreamUri: string;\n  readonly #shard: ShardID;\n  readonly #replicationConfig: ReplicationConfig;\n\n  constructor(\n    lc: LogContext,\n    upstreamUri: string,\n    shard: ShardID,\n    replicationConfig: ReplicationConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'change-source');\n    this.#upstreamUri = upstreamUri;\n    this.#shard = shard;\n    this.#replicationConfig = replicationConfig;\n  }\n\n  initialSync(): ChangeStream {\n    return this.#startStream();\n  }\n\n  startLagReporter() {\n    return null; // Not supported for custom sources\n  }\n\n  stop(): Promise<void> {\n    return Promise.resolve();\n  }\n\n  startStream(\n    clientWatermark: string,\n    backfillRequests: BackfillRequest[] = [],\n  ): Promise<ChangeStream> {\n    if (backfillRequests?.length) {\n      throw new Error(\n        'backfill is yet not supported for custom change sources',\n      );\n    }\n    return Promise.resolve(this.#startStream(clientWatermark));\n  }\n\n  #startStream(clientWatermark?: string): ChangeStream {\n    const {publications, replicaVersion} = this.#replicationConfig;\n    const {appID, shardNum} = this.#shard;\n    const url = new URL(this.#upstreamUri);\n    url.searchParams.set('appID', appID);\n    url.searchParams.set('shardNum', String(shardNum));\n    for (const pub of publications) {\n      url.searchParams.append('publications', pub);\n    }\n    if (clientWatermark) {\n      assert(\n        replicaVersion.length,\n        'replicaVersion is required when clientWatermark is set',\n      );\n      url.searchParams.set('lastWatermark', clientWatermark);\n      url.searchParams.set('replicaVersion', replicaVersion);\n    }\n\n    const ws = new WebSocket(url);\n    const {instream, outstream} = stream(\n      this.#lc,\n      ws,\n      changeStreamMessageSchema,\n      // Upstream acks coalesce. If upstream exhibits back-pressure,\n      // only the last ACK is kept / buffered.\n      {coalesce: (curr: ChangeSourceUpstream) => curr},\n    );\n    return {changes: instream, acks: outstream};\n  }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  context: ServerContext,\n) {\n  const {appID: id, publications} = shard;\n  const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n    replicaVersion: '', // ignored for initialSync()\n    publications,\n  });\n  const {changes} = changeSource.initialSync();\n\n  createReplicationStateTables(tx);\n  const processor = new ChangeProcessor(\n    new StatementRunner(tx),\n    'initial-sync',\n    (_, err) => {\n      throw err;\n    },\n  );\n\n  const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n  try {\n    let num = 0;\n    for await (const change of changes) {\n      const [tag] = change;\n      switch (tag) {\n        case 'begin': {\n          const {commitWatermark} = change[2];\n          lc.info?.(\n            `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n          );\n          statusPublisher.publish(\n            lc,\n            'Initializing',\n            `Copying upstream tables at version ${commitWatermark}`,\n            5000,\n          );\n          initReplicationState(\n            tx,\n            publications.toSorted(),\n            commitWatermark,\n            context,\n            false,\n          );\n          processor.processMessage(lc, change);\n          break;\n        }\n        case 'data':\n          processor.processMessage(lc, change);\n          if (++num % 1000 === 0) {\n            lc.debug?.(`processed ${num} changes`);\n          }\n          break;\n        case 'commit':\n          processor.processMessage(lc, change);\n          validateInitiallySyncedData(lc, tx, shard);\n          lc.info?.(`finished initial-sync of ${num} changes`);\n          return;\n\n        case 'status':\n          break; // Ignored\n        // @ts-expect-error: falls through if the tag is not 'reset-required\n        case 'control': {\n          const {tag, message} = change[1];\n          if (tag === 'reset-required') {\n            throw new AutoResetSignal(\n              message ?? 'auto-reset signaled by change source',\n            );\n          }\n        }\n        // falls through\n        case 'rollback':\n          throw new Error(\n            `unexpected message during initial-sync: ${stringify(change)}`,\n          );\n        default:\n          unreachable(change);\n      }\n    }\n    throw new Error(\n      `change source ${upstreamURI} closed before initial-sync completed`,\n    );\n  } catch (e) {\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n  }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n  appID,\n  shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n  return {\n    [`${appID}_${shardNum}.clients`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      lastMutationID: {type: 'number'},\n      userID: {type: 'string'},\n    },\n    [`${appID}_${shardNum}.mutations`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      mutationID: {type: 'number'},\n      mutation: {type: 'json'},\n    },\n    [`${appID}.permissions`]: {\n      permissions: {type: 'json'},\n      hash: {type: 'string'},\n    },\n  };\n}\n\nfunction validateInitiallySyncedData(\n  lc: LogContext,\n  db: Database,\n  shard: ShardID,\n) {\n  const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n  const required = getRequiredTables(shard);\n  for (const [name, columns] of Object.entries(required)) {\n    const table = tables.get(name)?.zqlSpec;\n    if (!table) {\n      throw new Error(\n        `Upstream is missing the \"${name}\" table. (Found ${[\n          ...tables.keys(),\n        ]})` +\n          `Please ensure that each table has a unique index over one ` +\n          `or more non-null columns.`,\n      );\n    }\n    for (const [col, {type}] of Object.entries(columns)) {\n      const found = table[col];\n      if (!found) {\n        throw new Error(\n          `Upstream \"${table}\" table is missing the \"${col}\" column`,\n        );\n      }\n      if (found.type !== type) {\n        throw new Error(\n          `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n        );\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA6CA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SACA,EAAC,YAAY,gBAA+B,CAAC,GAClB;CAS3B,IAAI,YAAY,WACd,MAAM,eAAe,IAAI,YAAY,eAAe,WAAW;CAGjE,IAAI,gBAAgB;CACpB,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO;EACX,gBAAgB;EAChB,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,OAAO;CACzD,CACF;CAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;CAC9C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;CAC3E,QAAQ,MAAM;CAEd,IAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,SAAS;EAC9C,MAAM,aAAa,kBAAkB,aAAa,KAAK;EACvD,IAAI,CAAC,UAAU,WAAW,UAAU,GAClC,MAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,EAC9G;CAEJ;CASA,OAAO;EACL;EACA,cAAA,IATuB,mBACvB,IACA,aACA,OACA,iBAKA;EACA,sBAAsB,YAAY;EAIlC,WAAW;EACX,4BAA4B;CAC9B;AACF;AAEA,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,eAAe;EACpB,KAAKC,SAAS;EACd,KAAKC,qBAAqB;CAC5B;CAEA,cAA4B;EAC1B,OAAO,KAAKC,aAAa;CAC3B;CAEA,mBAAmB;EACjB,OAAO;CACT;CAEA,OAAsB;EACpB,OAAO,QAAQ,QAAQ;CACzB;CAEA,YACE,iBACA,mBAAsC,CAAC,GAChB;EACvB,IAAI,kBAAkB,QACpB,MAAM,IAAI,MACR,yDACF;EAEF,OAAO,QAAQ,QAAQ,KAAKA,aAAa,eAAe,CAAC;CAC3D;CAEA,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,KAAKD;EAC5C,MAAM,EAAC,OAAO,aAAY,KAAKD;EAC/B,MAAM,MAAM,IAAI,IAAI,KAAKD,YAAY;EACrC,IAAI,aAAa,IAAI,SAAS,KAAK;EACnC,IAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;EACjD,KAAK,MAAM,OAAO,cAChB,IAAI,aAAa,OAAO,gBAAgB,GAAG;EAE7C,IAAI,iBAAiB;GACnB,OACE,eAAe,QACf,wDACF;GACA,IAAI,aAAa,IAAI,iBAAiB,eAAe;GACrD,IAAI,aAAa,IAAI,kBAAkB,cAAc;EACvD;EAEA,MAAM,KAAK,IAAI,UAAU,GAAG;EAC5B,MAAM,EAAC,UAAU,cAAa,OAC5B,KAAKD,KACL,IACA,2BAGA,EAAC,WAAW,SAA+B,KAAI,CACjD;EACA,OAAO;GAAC,SAAS;GAAU,MAAM;EAAS;CAC5C;AACF;;;;;;;;;;;;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAAW,IAJO,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;CACF,CACkB,EAAa,YAAY;CAE3C,6BAA6B,EAAE;CAC/B,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,EAAE,GACtB,iBACC,GAAG,QAAQ;EACV,MAAM;CACR,CACF;CAEA,MAAM,kBAAkB,2BAA2B,sBAAsB,EAAE;CAC3E,IAAI;EACF,IAAI,MAAM;EACV,WAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;GACd,QAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;KACjC,GAAG,OACD,yBAAyB,GAAG,qBAAqB,iBACnD;KACA,gBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,GACF;KACA,qBACE,IACA,aAAa,SAAS,GACtB,iBACA,SACA,KACF;KACA,UAAU,eAAe,IAAI,MAAM;KACnC;IACF;IACA,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,IAAI,EAAE,MAAM,QAAS,GACnB,GAAG,QAAQ,aAAa,IAAI,SAAS;KAEvC;IACF,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,4BAA4B,IAAI,IAAI,KAAK;KACzC,GAAG,OAAO,4BAA4B,IAAI,SAAS;KACnD;IAEF,KAAK,UACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;KAC9B,IAAI,QAAQ,kBACV,MAAM,IAAI,gBACR,WAAW,sCACb;IAEJ;IAEA,KAAK,YACH,MAAM,IAAI,MACR,2CAA2C,UAAU,MAAM,GAC7D;IACF,SACE,YAAY,MAAM;GACtB;EACF;EACA,MAAM,IAAI,MACR,iBAAiB,YAAY,sCAC/B;CACF,SAAS,GAAG;EACV,MAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;CAClE,UAAU;EACR,gBAAgB,KAAK;CACvB;AACF;AAIA,SAAS,kBAAkB,EACzB,OACA,YACuD;CACvD,OAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,gBAAgB,EAAC,MAAM,SAAQ;GAC/B,QAAQ,EAAC,MAAM,SAAQ;EACzB;GACC,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,YAAY,EAAC,MAAM,SAAQ;GAC3B,UAAU,EAAC,MAAM,OAAM;EACzB;GACC,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,OAAM;GAC1B,MAAM,EAAC,MAAM,SAAQ;EACvB;CACF;AACF;AAEA,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,KAAI,CAAC;CACxE,MAAM,WAAW,kBAAkB,KAAK;CACxC,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,QAAQ,GAAG;EACtD,MAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;EAChC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,KAAK,CACjB,EAAE,qFAGJ;EAEF,KAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,OAAO,GAAG;GACnD,MAAM,QAAQ,MAAM;GACpB,IAAI,CAAC,OACH,MAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,SACnD;GAEF,IAAI,MAAM,SAAS,MACjB,MAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,OAClF;EAEJ;CACF;AACF"}
+{"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n  stringify,\n  type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n  AutoResetSignal,\n  type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n  createReplicationStateTables,\n  getSubscriptionState,\n  initReplicationState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {\n  restoreReplica,\n  type InitializeResult,\n  type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n  type BackfillRequest,\n  type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  context: ServerContext,\n  {litestream, constraints}: RestoreOptions = {},\n): Promise<InitializeResult> {\n  // At the moment, the custom change-source implementation does not support\n  // per-RM backups (and thus does not support litestream v5 or RMv2). The\n  // current proposal to support this without requiring additional upstream\n  // state tracking is to:\n  // - use litestream to (ltx) query a fixed pool of subfolders within the\n  //   backupURL (e.g. /a/, /b/, /c/, /d/, /e/)\n  // - restore from the subfolder with the latest ltx file\n  // - backup to the subfolder with the oldest (or non-existent) ltx file\n  if (litestream?.backupURL) {\n    await restoreReplica(lc, litestream, replicaDbFile, constraints);\n  }\n\n  let initialSynced = false;\n  await initReplica(\n    lc,\n    `replica-${shard.appID}-${shard.shardNum}`,\n    replicaDbFile,\n    (log, tx) => {\n      initialSynced = true;\n      return initialSync(log, shard, tx, upstreamURI, context);\n    },\n  );\n\n  const replica = new Database(lc, replicaDbFile);\n  const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n  replica.close();\n\n  if (shard.publications.length) {\n    // Verify that the publications match what has been synced.\n    const requested = shard.publications.toSorted();\n    const replicated = subscriptionState.publications.sort();\n    if (!deepEqual(requested, replicated)) {\n      throw new Error(\n        `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n      );\n    }\n  }\n\n  const changeSource = new CustomChangeSource(\n    lc,\n    upstreamURI,\n    shard,\n    subscriptionState,\n  );\n\n  return {\n    subscriptionState,\n    changeSource,\n    destinationBackupURL: litestream?.backupURL,\n    // A custom change source has no upstream `replicas` table to identify\n    // itself from, so the generation is the whole of the SQLite change log's\n    // identity here.\n    replicaID: null,\n    waitForBackupBeforeServing: initialSynced,\n  };\n}\n\nclass CustomChangeSource implements ChangeSource {\n  readonly #lc: LogContext;\n  readonly #upstreamUri: string;\n  readonly #shard: ShardID;\n  readonly #replicationConfig: ReplicationConfig;\n\n  constructor(\n    lc: LogContext,\n    upstreamUri: string,\n    shard: ShardID,\n    replicationConfig: ReplicationConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'change-source');\n    this.#upstreamUri = upstreamUri;\n    this.#shard = shard;\n    this.#replicationConfig = replicationConfig;\n  }\n\n  initialSync(): ChangeStream {\n    return this.#startStream();\n  }\n\n  startLagReporter() {\n    return null; // Not supported for custom sources\n  }\n\n  stop(): Promise<void> {\n    return Promise.resolve();\n  }\n\n  startStream(\n    clientWatermark: string,\n    backfillRequests: BackfillRequest[] = [],\n  ): Promise<ChangeStream> {\n    if (backfillRequests?.length) {\n      throw new Error(\n        'backfill is yet not supported for custom change sources',\n      );\n    }\n    return Promise.resolve(this.#startStream(clientWatermark));\n  }\n\n  #startStream(clientWatermark?: string): ChangeStream {\n    const {publications, replicaVersion} = this.#replicationConfig;\n    const {appID, shardNum} = this.#shard;\n    const url = new URL(this.#upstreamUri);\n    url.searchParams.set('appID', appID);\n    url.searchParams.set('shardNum', String(shardNum));\n    for (const pub of publications) {\n      url.searchParams.append('publications', pub);\n    }\n    if (clientWatermark) {\n      assert(\n        replicaVersion.length,\n        'replicaVersion is required when clientWatermark is set',\n      );\n      url.searchParams.set('lastWatermark', clientWatermark);\n      url.searchParams.set('replicaVersion', replicaVersion);\n    }\n\n    const ws = new WebSocket(url);\n    const {instream, outstream} = stream(\n      this.#lc,\n      ws,\n      changeStreamMessageSchema,\n      // Upstream acks coalesce. If upstream exhibits back-pressure,\n      // only the last ACK is kept / buffered.\n      {coalesce: (curr: ChangeSourceUpstream) => curr},\n    );\n    return {changes: instream, acks: outstream};\n  }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  context: ServerContext,\n) {\n  const {appID: id, publications} = shard;\n  const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n    replicaVersion: '', // ignored for initialSync()\n    publications,\n  });\n  const {changes} = changeSource.initialSync();\n\n  createReplicationStateTables(tx);\n  const processor = new ChangeProcessor(\n    new StatementRunner(tx),\n    'initial-sync',\n    (_, err) => {\n      throw err;\n    },\n  );\n\n  const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n  try {\n    let num = 0;\n    for await (const change of changes) {\n      const [tag] = change;\n      switch (tag) {\n        case 'begin': {\n          const {commitWatermark} = change[2];\n          lc.info?.(\n            `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n          );\n          statusPublisher.publish(\n            lc,\n            'Initializing',\n            `Copying upstream tables at version ${commitWatermark}`,\n            5000,\n          );\n          initReplicationState(\n            tx,\n            publications.toSorted(),\n            commitWatermark,\n            context,\n            false,\n          );\n          processor.processMessage(lc, change);\n          break;\n        }\n        case 'data':\n          processor.processMessage(lc, change);\n          if (++num % 1000 === 0) {\n            lc.debug?.(`processed ${num} changes`);\n          }\n          break;\n        case 'commit':\n          processor.processMessage(lc, change);\n          validateInitiallySyncedData(lc, tx, shard);\n          lc.info?.(`finished initial-sync of ${num} changes`);\n          return;\n\n        case 'status':\n          break; // Ignored\n        // @ts-expect-error: falls through if the tag is not 'reset-required\n        case 'control': {\n          const {tag, message} = change[1];\n          if (tag === 'reset-required') {\n            throw new AutoResetSignal(\n              message ?? 'auto-reset signaled by change source',\n            );\n          }\n        }\n        // falls through\n        case 'rollback':\n          throw new Error(\n            `unexpected message during initial-sync: ${stringify(change)}`,\n          );\n        default:\n          unreachable(change);\n      }\n    }\n    throw new Error(\n      `change source ${upstreamURI} closed before initial-sync completed`,\n    );\n  } catch (e) {\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n  }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n  appID,\n  shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n  return {\n    [`${appID}_${shardNum}.clients`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      lastMutationID: {type: 'number'},\n      userID: {type: 'string'},\n    },\n    [`${appID}_${shardNum}.mutations`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      mutationID: {type: 'number'},\n      mutation: {type: 'json'},\n    },\n    [`${appID}.permissions`]: {\n      permissions: {type: 'json'},\n      hash: {type: 'string'},\n    },\n  };\n}\n\nfunction validateInitiallySyncedData(\n  lc: LogContext,\n  db: Database,\n  shard: ShardID,\n) {\n  const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n  const required = getRequiredTables(shard);\n  for (const [name, columns] of Object.entries(required)) {\n    const table = tables.get(name)?.zqlSpec;\n    if (!table) {\n      throw new Error(\n        `Upstream is missing the \"${name}\" table. (Found ${[\n          ...tables.keys(),\n        ]})` +\n          `Please ensure that each table has a unique index over one ` +\n          `or more non-null columns.`,\n      );\n    }\n    for (const [col, {type}] of Object.entries(columns)) {\n      const found = table[col];\n      if (!found) {\n        throw new Error(\n          `Upstream \"${table}\" table is missing the \"${col}\" column`,\n        );\n      }\n      if (found.type !== type) {\n        throw new Error(\n          `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n        );\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA6CA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SACA,EAAC,YAAY,gBAA+B,CAAC,GAClB;CAS3B,IAAI,YAAY,WACd,MAAM,eAAe,IAAI,YAAY,eAAe,WAAW;CAGjE,IAAI,gBAAgB;CACpB,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO;EACX,gBAAgB;EAChB,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,OAAO;CACzD,CACF;CAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;CAC9C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;CAC3E,QAAQ,MAAM;CAEd,IAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,SAAS;EAC9C,MAAM,aAAa,kBAAkB,aAAa,KAAK;EACvD,IAAI,CAAC,UAAU,WAAW,UAAU,GAClC,MAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,EAC9G;CAEJ;CASA,OAAO;EACL;EACA,cAAA,IATuB,mBACvB,IACA,aACA,OACA,iBAKA;EACA,sBAAsB,YAAY;EAIlC,WAAW;EACX,4BAA4B;CAC9B;AACF;AAEA,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,eAAe;EACpB,KAAKC,SAAS;EACd,KAAKC,qBAAqB;CAC5B;CAEA,cAA4B;EAC1B,OAAO,KAAKC,aAAa;CAC3B;CAEA,mBAAmB;EACjB,OAAO;CACT;CAEA,OAAsB;EACpB,OAAO,QAAQ,QAAQ;CACzB;CAEA,YACE,iBACA,mBAAsC,CAAC,GAChB;EACvB,IAAI,kBAAkB,QACpB,MAAM,IAAI,MACR,yDACF;EAEF,OAAO,QAAQ,QAAQ,KAAKA,aAAa,eAAe,CAAC;CAC3D;CAEA,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,KAAKD;EAC5C,MAAM,EAAC,OAAO,aAAY,KAAKD;EAC/B,MAAM,MAAM,IAAI,IAAI,KAAKD,YAAY;EACrC,IAAI,aAAa,IAAI,SAAS,KAAK;EACnC,IAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;EACjD,KAAK,MAAM,OAAO,cAChB,IAAI,aAAa,OAAO,gBAAgB,GAAG;EAE7C,IAAI,iBAAiB;GACnB,OACE,eAAe,QACf,wDACF;GACA,IAAI,aAAa,IAAI,iBAAiB,eAAe;GACrD,IAAI,aAAa,IAAI,kBAAkB,cAAc;EACvD;EAEA,MAAM,KAAK,IAAI,YAAU,GAAG;EAC5B,MAAM,EAAC,UAAU,cAAa,OAC5B,KAAKD,KACL,IACA,2BAGA,EAAC,WAAW,SAA+B,KAAI,CACjD;EACA,OAAO;GAAC,SAAS;GAAU,MAAM;EAAS;CAC5C;AACF;;;;;;;;;;;;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAAW,IAJO,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;CACF,CACkB,EAAa,YAAY;CAE3C,6BAA6B,EAAE;CAC/B,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,EAAE,GACtB,iBACC,GAAG,QAAQ;EACV,MAAM;CACR,CACF;CAEA,MAAM,kBAAkB,2BAA2B,sBAAsB,EAAE;CAC3E,IAAI;EACF,IAAI,MAAM;EACV,WAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;GACd,QAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;KACjC,GAAG,OACD,yBAAyB,GAAG,qBAAqB,iBACnD;KACA,gBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,GACF;KACA,qBACE,IACA,aAAa,SAAS,GACtB,iBACA,SACA,KACF;KACA,UAAU,eAAe,IAAI,MAAM;KACnC;IACF;IACA,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,IAAI,EAAE,MAAM,QAAS,GACnB,GAAG,QAAQ,aAAa,IAAI,SAAS;KAEvC;IACF,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,4BAA4B,IAAI,IAAI,KAAK;KACzC,GAAG,OAAO,4BAA4B,IAAI,SAAS;KACnD;IAEF,KAAK,UACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;KAC9B,IAAI,QAAQ,kBACV,MAAM,IAAI,gBACR,WAAW,sCACb;IAEJ;IAEA,KAAK,YACH,MAAM,IAAI,MACR,2CAA2C,UAAU,MAAM,GAC7D;IACF,SACE,YAAY,MAAM;GACtB;EACF;EACA,MAAM,IAAI,MACR,iBAAiB,YAAY,sCAC/B;CACF,SAAS,GAAG;EACV,MAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;CAClE,UAAU;EACR,gBAAgB,KAAK;CACvB;AACF;AAIA,SAAS,kBAAkB,EACzB,OACA,YACuD;CACvD,OAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,gBAAgB,EAAC,MAAM,SAAQ;GAC/B,QAAQ,EAAC,MAAM,SAAQ;EACzB;GACC,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,YAAY,EAAC,MAAM,SAAQ;GAC3B,UAAU,EAAC,MAAM,OAAM;EACzB;GACC,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,OAAM;GAC1B,MAAM,EAAC,MAAM,SAAQ;EACvB;CACF;AACF;AAEA,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,KAAI,CAAC;CACxE,MAAM,WAAW,kBAAkB,KAAK;CACxC,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,QAAQ,GAAG;EACtD,MAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;EAChC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,KAAK,CACjB,EAAE,qFAGJ;EAEF,KAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,OAAO,GAAG;GACnD,MAAM,QAAQ,MAAM;GACpB,IAAI,CAAC,OACH,MAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,SACnD;GAEF,IAAI,MAAM,SAAS,MACjB,MAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,OAClF;EAEJ;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -13,7 +13,7 @@ import { discoverChangeStreamerAddress } from "./schema/tables.js";
 import "./change-streamer.js";
 import "./snapshot-message.js";
 import { subscribeDownstreamSchema, subscribeUpstreamSchema } from "./subscribe.js";
-import WebSocket$1 from "ws";
+import WebSocket from "ws";
 import websocket from "@fastify/websocket";
 //#region ../zero-cache/src/services/change-streamer/change-streamer-http.ts
 var MIN_SUPPORTED_PROTOCOL_VERSION = 4;
@@ -206,7 +206,7 @@ var ChangeStreamerHttpClient = class {
 	* replication-manager is the one that serves the subscription.
 	*/
 	async connect(taskID) {
-		const ws = new WebSocket$1(await this.#resolveChangeStreamer(SUBSCRIBE_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
+		const ws = new WebSocket(await this.#resolveChangeStreamer(SUBSCRIBE_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
 		const outbound = Subscription.create();
 		return {
 			instream: await streamInternalWithSize(this.#lc, ws, subscribeDownstreamSchema, outbound),

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#changes","#reserveSnapshot","#subscribe","#receiveWebsocket","#ensureChangeStreamerStarted","#forwardReservation","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  readHead,\n  streamInternal,\n  streamInternalStringified,\n  streamInternalWithSize,\n  type Sized,\n  type Source,\n} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport {\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {type SnapshotMessage} from './snapshot-message.ts';\nimport {\n  subscribeDownstreamSchema,\n  subscribeUpstreamSchema,\n  type ReservedMessage,\n  type SnapshotStatus,\n  type SubscribeContext,\n  type SubscribeDownstream,\n  type SubscribeUpstream,\n} from './subscribe.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\n// The merged `/subscribe` protocol was introduced at v7.\nconst MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst SUBSCRIBE_PATH_PATTERN = '/replication/:version/subscribe';\nconst PATH_REGEX =\n  /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot|subscribe)$/;\n\nconst SUBSCRIBE_PATH = `/replication/v${PROTOCOL_VERSION}/subscribe`;\n\ntype Options = HttpOptions & {\n  startupDelayMs: number;\n  config?: Pick<NormalizedZeroConfig, 'adminPassword'> | undefined;\n  getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#changes);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n      fastify.get(SUBSCRIBE_PATH_PATTERN, {websocket: true}, this.#subscribe);\n\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(\n          lc,\n          opts.config ?? {adminPassword: undefined},\n          req,\n          res,\n          opts.getProfileWorker,\n          undefined,\n          'change-streamer',\n        ),\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes' | 'subscribe'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot' | 'subscribe',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#changes(ws, msg);\n      case 'subscribe':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      // Send-only for now: the reservation carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternal(this._lc, ws, undefined, downstream);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #changes = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      // Send-only for now: the subscription carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternalStringified(this._lc, ws, undefined, downstream, {\n        batched: ctx.wsBatched,\n      });\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  /**\n   * The merged v7 `/subscribe` handler. A single bidirectional connection\n   * carries, in sequence: an optional reservation phase (`reserve-snapshot` →\n   * `reserved`) and then the subscription phase (`start-subscription` → change\n   * stream). This ensures that the subscriber subscribes to the same task from\n   * which it restored the backup.\n   */\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    let reservation: Source<SnapshotMessage> | undefined;\n    const outbound = Subscription.create<string | PreSerializedBatch>();\n\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      const protocolVersion = checkProtocolVersion(url.pathname);\n      if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) {\n        throw new Error(\n          `the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} ` +\n            `(client is at v${protocolVersion})`,\n        );\n      }\n\n      const instream = await streamInternalStringified(\n        this._lc,\n        ws,\n        subscribeUpstreamSchema,\n        outbound,\n        {batched: true},\n      );\n\n      // Keep reading upstream control messages for the life of the connection.\n      // Do not manually exit loop, as that would close the socket. The loop\n      // exits naturally when the connection closes.\n      for await (const msg of instream) {\n        if (msg[0] === 'reserve-snapshot') {\n          const {taskID} = msg[1];\n          this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n          reservation =\n            await this.#changeStreamer.startSnapshotReservation(taskID);\n          void this.#forwardReservation(reservation, outbound);\n        } else if (msg[0] === 'start-subscription') {\n          const ctx: SubscriberContext = {\n            ...msg[1],\n            protocolVersion,\n            wsBatched: true,\n          };\n          if (ctx.mode === 'serving') {\n            this.#ensureChangeStreamerStarted('incoming subscription');\n          }\n          // subscribe() registers the subscriber and then releases any\n          // reservation for that task.\n          reservation = undefined;\n          await this.#changeStreamer.subscribe(ctx, outbound);\n        }\n      }\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    } finally {\n      // Covers a reservation-phase disconnect (instream ended before a\n      // subscription started); idempotent with the close handler above.\n      reservation?.cancel();\n    }\n  };\n\n  /**\n   * Forwards a snapshot reservation's single `['status', ...]` message onto\n   * the merged connection's `outbound` sink as a `['reserved', ...]` message.\n   * The loop then blocks on the held-open reservation until it is cancelled,\n   * by `subscribe()` at the subscription handoff, or by the reservation-phase\n   * teardown on disconnect.\n   */\n  async #forwardReservation(\n    reservation: Source<SnapshotMessage>,\n    outbound: Subscription<string | PreSerializedBatch>,\n  ) {\n    try {\n      for await (const [, status] of reservation) {\n        // Translate the /snapshot protocol's ['status', {tag: 'status', ...}]\n        // message to the /subscribe protocol's ['reserved', {tag: 'snapshot', ...}].\n        outbound.push(\n          BigIntJSON.stringify([\n            'reserved',\n            {...status, tag: 'snapshot'},\n          ] satisfies ReservedMessage),\n        );\n      }\n    } catch (e) {\n      this._lc.warn?.(`error forwarding snapshot reservation`, e);\n      outbound.fail(e instanceof Error ? e : new Error(String(e)));\n    }\n  }\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason?: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(\n        `starting ChangeStreamerService ${reason ? `(${reason})` : ''}`,\n      );\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n    if (startupDelayMs === 0) {\n      // In RMv2, there is no need to delay starting the change streamer\n      // because RM startup is non-disruptive.\n      this.#ensureChangeStreamerStarted();\n    } else {\n      // In RMv1, starting the change-streamer forcibly shuts down the\n      // previous change-streamer, causing view-syncers to reconnect.\n      // If this replication-manager has just started, the routing layer may\n      // not have registered it with DNS, as that only happens after it\n      // confirms health checks. To minimize downtime, the takeover is\n      // delayed for the configured startupDelayMs _after_ beginning to\n      // advertise readiness.\n      void readinessGate.then(() => {\n        this.#lc.info?.(\n          `waiting ${startupDelayMs}ms before taking over the change log`,\n        );\n        this._state.setTimeout(\n          () =>\n            this.#ensureChangeStreamerStarted(\n              `startup delay elapsed (${startupDelayMs} ms)`,\n            ),\n          startupDelayMs,\n        );\n      });\n    }\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport interface SnapshotReserver {\n  reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }>;\n}\n\nexport interface SubscriptionStarter {\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\nexport interface ReservationFollowup extends SubscriptionStarter {\n  /**\n   * A signal to indicate that the followup is no longer valid (i.e. the\n   * connection to the reserving task has been severed).\n   */\n  readonly signal: AbortSignal;\n\n  /**\n   * Abandons the reservation without subscribing, closing the connection that\n   * was held open to pin the change-log floor. Used when the caller cannot\n   * proceed to `subscribe()` (e.g. process shutdown while a restore is in\n   * progress) and must not leave the reservation's connection open forever.\n   */\n  cancel(reason?: Error): void;\n}\n\nexport class ChangeStreamerHttpClient\n  implements ChangeStreamer, SnapshotReserver, SubscriptionStarter\n{\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }> {\n    const {instream, outbound} = await this.connect(taskID);\n    outbound.push(['reserve-snapshot', {taskID}]);\n\n    const {head, rest} = readHead(instream);\n    const item = await head.next();\n    if (!item.value) {\n      throw new Error('stream closed without snapshot reservation');\n    }\n    const {data: msg} = item.value;\n    if (msg[0] !== 'reserved') {\n      throw new Error(`expected \"reserved\" message but got ${msg[0]}`);\n    }\n    const reserved: SnapshotStatus = msg[1];\n    const followup: ReservationFollowup = {\n      subscribe: ctx => {\n        outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n        return Promise.resolve(rest);\n      },\n      signal: outbound.signal,\n      cancel: reason => outbound.cancel(reason),\n    };\n\n    return {reserved, followup};\n  }\n\n  async subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>> {\n    const {instream, outbound} = await this.connect(ctx.taskID);\n    outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n    return instream;\n  }\n\n  /**\n   * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the\n   * inbound change/reservation stream and an `outbound` sink into which the\n   * caller pushes the `reserve-snapshot` / `start-subscription` control\n   * messages that drive the connection's phases. The single connection spans\n   * the reservation, backup restore, and subscription, so the reserving\n   * replication-manager is the one that serves the subscription.\n   */\n  // exported for testing\n  async connect(taskID: string): Promise<{\n    instream: Source<Sized<SubscribeDownstream>>;\n    outbound: Subscription<SubscribeUpstream>;\n  }> {\n    const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);\n\n    // taskID is carried in the query for observability/routing; the server\n    // reads the authoritative taskID from the control messages themselves.\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    const outbound = Subscription.create<SubscribeUpstream>();\n    const instream = await streamInternalWithSize(\n      this.#lc,\n      ws,\n      subscribeDownstreamSchema,\n      outbound,\n    );\n    return {instream, outbound};\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    wsBatched: params.getBoolean('wsBatched'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// Projects a client-side SubscriberContext onto the start-subscription\n// message payload, dropping the fields that are implicit on the merged\n// connection: protocolVersion (from the request path), wsBatched (always on).\nfunction toStartSubscriptionContext(ctx: SubscriberContext): SubscribeContext {\n  const {taskID, id, mode, replicaVersion, watermark} = ctx;\n  return {taskID, id, mode, replicaVersion, watermark};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,IAAM,iCAAiC;AAGvC,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAC/B,IAAM,aACJ;AAEF,IAAM,iBAAiB;AAQvB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,QAAQ;GAClE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GACA,QAAQ,IAAI,wBAAwB,EAAC,WAAW,KAAI,GAAG,KAAKC,UAAU;GAEtE,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU,EAAC,eAAe,KAAA,EAAS,GACxC,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,SAAS,IAAI,GAAG;GAC9B,KAAK,aACH,OAAO,KAAKE,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKE,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;GAG5D,eAAoB,KAAK,KAAK,IAAI,KAAA,GAAW,UAAU;EACzD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,WAAoB,OAAO,IAAe,QAAwB;EAChE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKK,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKL,gBAAgB,UAAU,GAAG;GAG3D,0BAA+B,KAAK,KAAK,IAAI,KAAA,GAAW,YAAY,EAClE,SAAS,IAAI,UACf,CAAC;EACH,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;;;;;;;;CASA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;EACJ,MAAM,WAAW,aAAa,OAAoC;EAElE,IAAI;GAKF,MAAM,kBAAkB,qBAAqB,IAJ7B,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBAEqB,EAAI,QAAQ;GACzD,IAAI,kBAAkB,gCACpB,MAAM,IAAI,MACR,8CAA8C,+BAA+B,kBACzD,gBAAgB,EACtC;GAGF,MAAM,WAAW,MAAM,0BACrB,KAAK,KACL,IACA,yBACA,UACA,EAAC,SAAS,KAAI,CAChB;GAKA,WAAW,MAAM,OAAO,UACtB,IAAI,IAAI,OAAO,oBAAoB;IACjC,MAAM,EAAC,WAAU,IAAI;IACrB,KAAKK,6BAA6B,+BAA+B;IACjE,cACE,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;IAC5D,KAAUM,oBAAoB,aAAa,QAAQ;GACrD,OAAO,IAAI,IAAI,OAAO,sBAAsB;IAC1C,MAAM,MAAyB;KAC7B,GAAG,IAAI;KACP;KACA,WAAW;IACb;IACA,IAAI,IAAI,SAAS,WACf,KAAKD,6BAA6B,uBAAuB;IAI3D,cAAc,KAAA;IACd,MAAM,KAAKL,gBAAgB,UAAU,KAAK,QAAQ;GACpD;EAEJ,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD,UAAU;GAGR,aAAa,OAAO;EACtB;CACF;;;;;;;;CASA,MAAMM,oBACJ,aACA,UACA;EACA,IAAI;GACF,WAAW,MAAM,GAAG,WAAW,aAG7B,SAAS,KACP,WAAW,UAAU,CACnB,YACA;IAAC,GAAG;IAAQ,KAAK;GAAU,CAC7B,CAA2B,CAC7B;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,SAAS,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EAC7D;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAiB;EAC5C,IAAI,CAAC,KAAKC,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OACP,kCAAkC,SAAS,IAAI,OAAO,KAAK,IAC7D;GACA,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKR;EAC3D,IAAI,mBAAmB,GAGrB,KAAKM,6BAA6B;OASlC,cAAmB,WAAW;GAC5B,KAAKP,IAAI,OACP,WAAW,eAAe,qCAC5B;GACA,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;EACF,CAAC;CAEL;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKE,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AA+BA,IAAa,2BAAb,MAEA;CACE;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAGnB;EACD,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,MAAM;EACtD,SAAS,KAAK,CAAC,oBAAoB,EAAC,OAAM,CAAC,CAAC;EAE5C,MAAM,EAAC,MAAM,SAAQ,SAAS,QAAQ;EACtC,MAAM,OAAO,MAAM,KAAK,KAAK;EAC7B,IAAI,CAAC,KAAK,OACR,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,EAAC,MAAM,QAAO,KAAK;EACzB,IAAI,IAAI,OAAO,YACb,MAAM,IAAI,MAAM,uCAAuC,IAAI,IAAI;EAYjE,OAAO;GAAC,UAVyB,IAAI;GAUnB,UAAA;IARhB,YAAW,QAAO;KAChB,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;KACrE,OAAO,QAAQ,QAAQ,IAAI;IAC7B;IACA,QAAQ,SAAS;IACjB,SAAQ,WAAU,SAAS,OAAO,MAAM;GAGxB;EAAQ;CAC5B;CAEA,MAAM,UACJ,KAC6C;EAC7C,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,IAAI,MAAM;EAC1D,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;EACrE,OAAO;CACT;;;;;;;;;CAWA,MAAM,QAAQ,QAGX;EAMD,MAAM,KAAK,IAAI,YAAU,MALP,KAAKa,uBAAuB,cAAc,IAK7B,IAAI,IADhB,gBAAgB,EAAC,OAAM,CACP,EAAO,SAAS,GAAG;EAEtD,MAAM,WAAW,aAAa,OAA0B;EAOxD,OAAO;GAAC,UAAA,MANe,uBACrB,KAAKb,KACL,IACA,2BACA,QACF;GACkB;EAAQ;CAC5B;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EAIvC,WAAW,OAAO,WAAW,WAAW;CAC1C;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAKA,SAAS,2BAA2B,KAA0C;CAC5E,MAAM,EAAC,QAAQ,IAAI,MAAM,gBAAgB,cAAa;CACtD,OAAO;EAAC;EAAQ;EAAI;EAAM;EAAgB;CAAS;AACrD"}
+{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#changes","#reserveSnapshot","#subscribe","#receiveWebsocket","#ensureChangeStreamerStarted","#forwardReservation","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  readHead,\n  streamInternal,\n  streamInternalStringified,\n  streamInternalWithSize,\n  type Sized,\n  type Source,\n} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport {\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {type SnapshotMessage} from './snapshot-message.ts';\nimport {\n  subscribeDownstreamSchema,\n  subscribeUpstreamSchema,\n  type ReservedMessage,\n  type SnapshotStatus,\n  type SubscribeContext,\n  type SubscribeDownstream,\n  type SubscribeUpstream,\n} from './subscribe.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\n// The merged `/subscribe` protocol was introduced at v7.\nconst MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst SUBSCRIBE_PATH_PATTERN = '/replication/:version/subscribe';\nconst PATH_REGEX =\n  /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot|subscribe)$/;\n\nconst SUBSCRIBE_PATH = `/replication/v${PROTOCOL_VERSION}/subscribe`;\n\ntype Options = HttpOptions & {\n  startupDelayMs: number;\n  config?: Pick<NormalizedZeroConfig, 'adminPassword'> | undefined;\n  getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#changes);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n      fastify.get(SUBSCRIBE_PATH_PATTERN, {websocket: true}, this.#subscribe);\n\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(\n          lc,\n          opts.config ?? {adminPassword: undefined},\n          req,\n          res,\n          opts.getProfileWorker,\n          undefined,\n          'change-streamer',\n        ),\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes' | 'subscribe'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot' | 'subscribe',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#changes(ws, msg);\n      case 'subscribe':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      // Send-only for now: the reservation carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternal(this._lc, ws, undefined, downstream);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #changes = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      // Send-only for now: the subscription carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternalStringified(this._lc, ws, undefined, downstream, {\n        batched: ctx.wsBatched,\n      });\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  /**\n   * The merged v7 `/subscribe` handler. A single bidirectional connection\n   * carries, in sequence: an optional reservation phase (`reserve-snapshot` →\n   * `reserved`) and then the subscription phase (`start-subscription` → change\n   * stream). This ensures that the subscriber subscribes to the same task from\n   * which it restored the backup.\n   */\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    let reservation: Source<SnapshotMessage> | undefined;\n    const outbound = Subscription.create<string | PreSerializedBatch>();\n\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      const protocolVersion = checkProtocolVersion(url.pathname);\n      if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) {\n        throw new Error(\n          `the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} ` +\n            `(client is at v${protocolVersion})`,\n        );\n      }\n\n      const instream = await streamInternalStringified(\n        this._lc,\n        ws,\n        subscribeUpstreamSchema,\n        outbound,\n        {batched: true},\n      );\n\n      // Keep reading upstream control messages for the life of the connection.\n      // Do not manually exit loop, as that would close the socket. The loop\n      // exits naturally when the connection closes.\n      for await (const msg of instream) {\n        if (msg[0] === 'reserve-snapshot') {\n          const {taskID} = msg[1];\n          this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n          reservation =\n            await this.#changeStreamer.startSnapshotReservation(taskID);\n          void this.#forwardReservation(reservation, outbound);\n        } else if (msg[0] === 'start-subscription') {\n          const ctx: SubscriberContext = {\n            ...msg[1],\n            protocolVersion,\n            wsBatched: true,\n          };\n          if (ctx.mode === 'serving') {\n            this.#ensureChangeStreamerStarted('incoming subscription');\n          }\n          // subscribe() registers the subscriber and then releases any\n          // reservation for that task.\n          reservation = undefined;\n          await this.#changeStreamer.subscribe(ctx, outbound);\n        }\n      }\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    } finally {\n      // Covers a reservation-phase disconnect (instream ended before a\n      // subscription started); idempotent with the close handler above.\n      reservation?.cancel();\n    }\n  };\n\n  /**\n   * Forwards a snapshot reservation's single `['status', ...]` message onto\n   * the merged connection's `outbound` sink as a `['reserved', ...]` message.\n   * The loop then blocks on the held-open reservation until it is cancelled,\n   * by `subscribe()` at the subscription handoff, or by the reservation-phase\n   * teardown on disconnect.\n   */\n  async #forwardReservation(\n    reservation: Source<SnapshotMessage>,\n    outbound: Subscription<string | PreSerializedBatch>,\n  ) {\n    try {\n      for await (const [, status] of reservation) {\n        // Translate the /snapshot protocol's ['status', {tag: 'status', ...}]\n        // message to the /subscribe protocol's ['reserved', {tag: 'snapshot', ...}].\n        outbound.push(\n          BigIntJSON.stringify([\n            'reserved',\n            {...status, tag: 'snapshot'},\n          ] satisfies ReservedMessage),\n        );\n      }\n    } catch (e) {\n      this._lc.warn?.(`error forwarding snapshot reservation`, e);\n      outbound.fail(e instanceof Error ? e : new Error(String(e)));\n    }\n  }\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason?: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(\n        `starting ChangeStreamerService ${reason ? `(${reason})` : ''}`,\n      );\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n    if (startupDelayMs === 0) {\n      // In RMv2, there is no need to delay starting the change streamer\n      // because RM startup is non-disruptive.\n      this.#ensureChangeStreamerStarted();\n    } else {\n      // In RMv1, starting the change-streamer forcibly shuts down the\n      // previous change-streamer, causing view-syncers to reconnect.\n      // If this replication-manager has just started, the routing layer may\n      // not have registered it with DNS, as that only happens after it\n      // confirms health checks. To minimize downtime, the takeover is\n      // delayed for the configured startupDelayMs _after_ beginning to\n      // advertise readiness.\n      void readinessGate.then(() => {\n        this.#lc.info?.(\n          `waiting ${startupDelayMs}ms before taking over the change log`,\n        );\n        this._state.setTimeout(\n          () =>\n            this.#ensureChangeStreamerStarted(\n              `startup delay elapsed (${startupDelayMs} ms)`,\n            ),\n          startupDelayMs,\n        );\n      });\n    }\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport interface SnapshotReserver {\n  reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }>;\n}\n\nexport interface SubscriptionStarter {\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\nexport interface ReservationFollowup extends SubscriptionStarter {\n  /**\n   * A signal to indicate that the followup is no longer valid (i.e. the\n   * connection to the reserving task has been severed).\n   */\n  readonly signal: AbortSignal;\n\n  /**\n   * Abandons the reservation without subscribing, closing the connection that\n   * was held open to pin the change-log floor. Used when the caller cannot\n   * proceed to `subscribe()` (e.g. process shutdown while a restore is in\n   * progress) and must not leave the reservation's connection open forever.\n   */\n  cancel(reason?: Error): void;\n}\n\nexport class ChangeStreamerHttpClient\n  implements ChangeStreamer, SnapshotReserver, SubscriptionStarter\n{\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }> {\n    const {instream, outbound} = await this.connect(taskID);\n    outbound.push(['reserve-snapshot', {taskID}]);\n\n    const {head, rest} = readHead(instream);\n    const item = await head.next();\n    if (!item.value) {\n      throw new Error('stream closed without snapshot reservation');\n    }\n    const {data: msg} = item.value;\n    if (msg[0] !== 'reserved') {\n      throw new Error(`expected \"reserved\" message but got ${msg[0]}`);\n    }\n    const reserved: SnapshotStatus = msg[1];\n    const followup: ReservationFollowup = {\n      subscribe: ctx => {\n        outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n        return Promise.resolve(rest);\n      },\n      signal: outbound.signal,\n      cancel: reason => outbound.cancel(reason),\n    };\n\n    return {reserved, followup};\n  }\n\n  async subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>> {\n    const {instream, outbound} = await this.connect(ctx.taskID);\n    outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n    return instream;\n  }\n\n  /**\n   * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the\n   * inbound change/reservation stream and an `outbound` sink into which the\n   * caller pushes the `reserve-snapshot` / `start-subscription` control\n   * messages that drive the connection's phases. The single connection spans\n   * the reservation, backup restore, and subscription, so the reserving\n   * replication-manager is the one that serves the subscription.\n   */\n  // exported for testing\n  async connect(taskID: string): Promise<{\n    instream: Source<Sized<SubscribeDownstream>>;\n    outbound: Subscription<SubscribeUpstream>;\n  }> {\n    const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);\n\n    // taskID is carried in the query for observability/routing; the server\n    // reads the authoritative taskID from the control messages themselves.\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    const outbound = Subscription.create<SubscribeUpstream>();\n    const instream = await streamInternalWithSize(\n      this.#lc,\n      ws,\n      subscribeDownstreamSchema,\n      outbound,\n    );\n    return {instream, outbound};\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    wsBatched: params.getBoolean('wsBatched'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// Projects a client-side SubscriberContext onto the start-subscription\n// message payload, dropping the fields that are implicit on the merged\n// connection: protocolVersion (from the request path), wsBatched (always on).\nfunction toStartSubscriptionContext(ctx: SubscriberContext): SubscribeContext {\n  const {taskID, id, mode, replicaVersion, watermark} = ctx;\n  return {taskID, id, mode, replicaVersion, watermark};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,IAAM,iCAAiC;AAGvC,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAC/B,IAAM,aACJ;AAEF,IAAM,iBAAiB;AAQvB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,QAAQ;GAClE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GACA,QAAQ,IAAI,wBAAwB,EAAC,WAAW,KAAI,GAAG,KAAKC,UAAU;GAEtE,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU,EAAC,eAAe,KAAA,EAAS,GACxC,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,SAAS,IAAI,GAAG;GAC9B,KAAK,aACH,OAAO,KAAKE,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKE,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;GAG5D,eAAoB,KAAK,KAAK,IAAI,KAAA,GAAW,UAAU;EACzD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,WAAoB,OAAO,IAAe,QAAwB;EAChE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKK,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKL,gBAAgB,UAAU,GAAG;GAG3D,0BAA+B,KAAK,KAAK,IAAI,KAAA,GAAW,YAAY,EAClE,SAAS,IAAI,UACf,CAAC;EACH,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;;;;;;;;CASA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;EACJ,MAAM,WAAW,aAAa,OAAoC;EAElE,IAAI;GAKF,MAAM,kBAAkB,qBAAqB,IAJ7B,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBAEqB,EAAI,QAAQ;GACzD,IAAI,kBAAkB,gCACpB,MAAM,IAAI,MACR,8CAA8C,+BAA+B,kBACzD,gBAAgB,EACtC;GAGF,MAAM,WAAW,MAAM,0BACrB,KAAK,KACL,IACA,yBACA,UACA,EAAC,SAAS,KAAI,CAChB;GAKA,WAAW,MAAM,OAAO,UACtB,IAAI,IAAI,OAAO,oBAAoB;IACjC,MAAM,EAAC,WAAU,IAAI;IACrB,KAAKK,6BAA6B,+BAA+B;IACjE,cACE,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;IAC5D,KAAUM,oBAAoB,aAAa,QAAQ;GACrD,OAAO,IAAI,IAAI,OAAO,sBAAsB;IAC1C,MAAM,MAAyB;KAC7B,GAAG,IAAI;KACP;KACA,WAAW;IACb;IACA,IAAI,IAAI,SAAS,WACf,KAAKD,6BAA6B,uBAAuB;IAI3D,cAAc,KAAA;IACd,MAAM,KAAKL,gBAAgB,UAAU,KAAK,QAAQ;GACpD;EAEJ,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD,UAAU;GAGR,aAAa,OAAO;EACtB;CACF;;;;;;;;CASA,MAAMM,oBACJ,aACA,UACA;EACA,IAAI;GACF,WAAW,MAAM,GAAG,WAAW,aAG7B,SAAS,KACP,WAAW,UAAU,CACnB,YACA;IAAC,GAAG;IAAQ,KAAK;GAAU,CAC7B,CAA2B,CAC7B;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,SAAS,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EAC7D;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAiB;EAC5C,IAAI,CAAC,KAAKC,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OACP,kCAAkC,SAAS,IAAI,OAAO,KAAK,IAC7D;GACA,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKR;EAC3D,IAAI,mBAAmB,GAGrB,KAAKM,6BAA6B;OASlC,cAAmB,WAAW;GAC5B,KAAKP,IAAI,OACP,WAAW,eAAe,qCAC5B;GACA,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;EACF,CAAC;CAEL;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKE,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AA+BA,IAAa,2BAAb,MAEA;CACE;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAGnB;EACD,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,MAAM;EACtD,SAAS,KAAK,CAAC,oBAAoB,EAAC,OAAM,CAAC,CAAC;EAE5C,MAAM,EAAC,MAAM,SAAQ,SAAS,QAAQ;EACtC,MAAM,OAAO,MAAM,KAAK,KAAK;EAC7B,IAAI,CAAC,KAAK,OACR,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,EAAC,MAAM,QAAO,KAAK;EACzB,IAAI,IAAI,OAAO,YACb,MAAM,IAAI,MAAM,uCAAuC,IAAI,IAAI;EAYjE,OAAO;GAAC,UAVyB,IAAI;GAUnB,UAAA;IARhB,YAAW,QAAO;KAChB,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;KACrE,OAAO,QAAQ,QAAQ,IAAI;IAC7B;IACA,QAAQ,SAAS;IACjB,SAAQ,WAAU,SAAS,OAAO,MAAM;GAGxB;EAAQ;CAC5B;CAEA,MAAM,UACJ,KAC6C;EAC7C,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,IAAI,MAAM;EAC1D,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;EACrE,OAAO;CACT;;;;;;;;;CAWA,MAAM,QAAQ,QAGX;EAMD,MAAM,KAAK,IAAI,UAAU,MALP,KAAKa,uBAAuB,cAAc,IAK7B,IAAI,IADhB,gBAAgB,EAAC,OAAM,CACP,EAAO,SAAS,GAAG;EAEtD,MAAM,WAAW,aAAa,OAA0B;EAOxD,OAAO;GAAC,UAAA,MANe,uBACrB,KAAKb,KACL,IACA,2BACA,QACF;GACkB;EAAQ;CAC5B;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EAIvC,WAAW,OAAO,WAAW,WAAW;CAC1C;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAKA,SAAS,2BAA2B,KAA0C;CAC5E,MAAM,EAAC,QAAQ,IAAI,MAAM,gBAAgB,cAAa;CACtD,OAAO;EAAC;EAAQ;EAAI;EAAM;EAAgB;CAAS;AACrD"}

package/out/zero-cache/src/services/replicator/change-processor.js

@@ -465,6 +465,11 @@ var TransactionProcessor = class {
 		const tableSpec = must(this.#tableSpecs.get(tableName));
 		const rowKeyCols = relation.rowKey.columns;
 		const cols = [...rowKeyCols, ...columns];
+		const backfillingSet = new Set(tableSpec.backfilling ?? []);
+		if (!cols.some((c) => backfillingSet.has(c))) {
+			this.#lc.debug?.(`skipping redundant backfill of ${rowValues.length} rows into ${tableName} (all columns already published)`);
+			return;
+		}
 		const insertColsStr = [...cols, ZERO_VERSION_COLUMN_NAME].map(id).join(",");
 		const qMarks = Array.from({ length: cols.length + 1 }).fill("?").join(",");
 		const rowKeyColsStr = rowKeyCols.map(id).join(",");
@@ -478,7 +483,7 @@ var TransactionProcessor = class {
 				skipped++;
 				continue;
 			}
-			const updates = rowOp?.op === "s" ? cols.filter((c) => (rowOp.backfillingColumnVersions[c] ?? "") <= watermark) : cols;
+			const updates = (rowOp?.op === "s" ? cols.filter((c) => (rowOp.backfillingColumnVersions[c] ?? "") <= watermark) : cols).filter((c) => backfillingSet.has(c));
 			if (updates.length === 0) {
 				skipped++;
 				continue;
@@ -498,6 +503,12 @@ var TransactionProcessor = class {
 		const { relation, columns, status } = msg;
 		const tableName = liteTableName(relation);
 		const cols = [...relation.rowKey.columns, ...columns];
+		const tableSpec = must(this.#tableSpecs.get(tableName));
+		const backfillingSet = new Set(tableSpec.backfilling ?? []);
+		if (!cols.some((col) => backfillingSet.has(col))) {
+			this.#lc.debug?.(`skipping redundant backfill-completed for ${tableName} (all columns already published)`);
+			return;
+		}
 		const columnMetadata = must(ColumnMetadataStore.getInstance(this.#db.db));
 		for (const col of cols) columnMetadata.clearBackfilling(tableName, col);
 		this.#backfilling.apply(msg);