@rocicorp/zero 0.26.1 → 0.26.2-canary.1

package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-manager.js","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<MessageBackfill | BackfillCompleted>;\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  canceledReason?: string | undefined;\n  minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {request, minWatermark: ''};\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    const log = this.#retryDelayMs === this.#maxBackoffMs ? 'error' : 'warn';\n    this.#lc[log]?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.canceledReason ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (state.canceledReason === undefined) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      const tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n      }\n      backfillTx = null;\n    };\n\n    for await (const msg of this.#backfillStreamer(state.request)) {\n      // Before sending `backfill-completed`, the main replication stream\n      // may need to catch up.\n      const mustWaitBeforeFlush =\n        msg.tag === 'backfill-completed' &&\n        this.#changeStreamReached(lc, msg.watermark);\n\n      // If necessary, yield the reservation to the main stream.\n      if (\n        backfillTx &&\n        (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush)\n      ) {\n        commitTx();\n      }\n\n      mustWaitBeforeFlush && (await mustWaitBeforeFlush);\n\n      if (\n        msg.tag === 'backfill' &&\n        msg.rowValues.length > 0 &&\n        msg.relation.rowKey.columns.length === 0\n      ) {\n        throw new MissingRowKeyError(state.request);\n      }\n\n      // Reserve the changeStreamer if not in a transaction.\n      if ((backfillTx ??= await beginTxFor(msg)) === null) {\n        lc.info?.(\n          `backfill stream canceled: ${state.canceledReason}`,\n          state.request,\n        );\n        this.#checkAndStartBackfill(); // start the next backfill if present\n        return; // this backfill is canceled\n      }\n\n      // `await` to allow the change streamer to exert back pressure\n      // on backfills.\n      await changeStream.push(['data', msg]);\n    }\n\n    // Flush any final tx and release the stream.\n    backfillTx && commitTx();\n    lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.canceledReason = reason;\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: backfill,\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: backfill},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: backfill,\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"names":[],"mappings":";;;;;;;;AAyBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;AAC5C,SAAO,GAAG,MAAM,IAAI,IAAI;AAC1B;AAYA,MAAM,0BAA0B;AAChC,MAAM,0BAA0B;AAuBzB,MAAM,gBAAgD;AAAA,EAClD;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,qBAAqB,IAAI;AAAA,IAChC;AAAA,EAAA;AAAA,EAEO;AAAA,EACA;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQT,mBAAgD;AAAA;AAAA,EAGhD,uBAAsC;AAAA,EAE7B,4BAAuD,CAAA;AAAA;AAAA,EAGhE,sBAAqC;AAAA,EAErC,YACE,IACA,gBACA,kBACA,aAAyB,kBACzB,eAAe,yBACf,eAAe,yBACf;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,kBAAkB;AACzD,SAAK,kBAAkB;AACvB,SAAK,oBAAoB;AACzB,SAAK,cAAc;AACnB,SAAK,gBAAgB;AACrB,SAAK,gBAAgB;AACrB,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAuB,iBAAoC;AAC7D,SAAK,IAAI;AAAA,MACP,kCAAkC,gBAAgB,MAAM;AAAA,MACxD,EAAC,UAAU,gBAAA;AAAA,IAAe;AAE5B,SAAK,uBAAuB;AAC5B,oBAAgB;AAAA,MAAQ,CAAA,QACtB,KAAK,qBAAqB,mBAAmB,GAAG;AAAA,IAAA;AAElD,SAAK,uBAAA;AAAA,EACP;AAAA,EAEA,wBAAwB,EAAC,aAAiC;AAIxD,SAAK,KAAK,wBAAwB,MAAM,WAAW;AACjD,WAAK,uBAAuB;AAC5B,eAAS,IAAI,KAAK,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;AACnE,cAAM,WAAW,KAAK,0BAA0B,CAAC;AACjD,YAAI,aAAa,SAAS,WAAW;AACnC,mBAAS,QAAA;AACT,eAAK,0BAA0B,OAAO,GAAG,CAAC;AAAA,QAC5C;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,qBACE,IACA,WACsB;AACtB,SAAK,KAAK,wBAAwB,MAAM,WAAW;AACjD,YAAM,EAAC,SAAS,SAAS,QAAA,IAAW,SAAA;AACpC,WAAK,0BAA0B,KAAK,EAAC,WAAW,SAAQ;AACxD,SAAG;AAAA,QACD,iCAAiC,KAAK,oBAAoB,cAAc,SAAS;AAAA,MAAA;AAEnF,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAAA,EAES;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EAEA,yBAAyB;AACvB,QACE,CAAC,KAAK,uBACN,CAAC,KAAK,oBACN,KAAK,mBAAmB,MACxB;AAKA,YAAM,aAAa,CAAC,GAAG,KAAK,mBAAmB,QAAQ;AACvD,YAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC,CAAC;AAC5D,YAAM,QAAQ,EAAC,SAAS,cAAc,GAAA;AACtC,YAAM,KAAK,KAAK,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;AAE3D,WAAK,mBAAmB;AACxB,WAAK,KAAK,aAAa,IAAI,KAAK,EAC7B,KAAK,MAAM;AACV,aAAK,qBAAqB,mBAAmB,KAAK;AAClD,aAAK,gBAAgB,KAAK;AAAA,MAC5B,CAAC,EAGA,MAAM,CAAA,MAAK;AACV,aAAK,qBAAqB,OAAO,CAAC,GAAG,KAAK;AAC1C,aAAK,0BAA0B,CAAC;AAAA,MAClC,CAAC;AAAA,IACL;AAAA,EACF;AAAA,EAEA,0BAA0B,GAAY;AACpC,UAAM,MAAM,KAAK,kBAAkB,KAAK,gBAAgB,UAAU;AAClE,SAAK,IAAI,GAAG;AAAA,MACV,uCAAuC,KAAK,aAAa;AAAA,MACzD;AAAA,IAAA;AAEF,SAAK,sBAAsB,WAAW,MAAM;AAC1C,WAAK,sBAAsB;AAC3B,WAAK,uBAAA;AAAA,IACP,GAAG,KAAK,aAAa;AAErB,SAAK,gBAAgB,KAAK,IAAI,KAAK,gBAAgB,GAAG,KAAK,aAAa;AAAA,EAC1E;AAAA,EAEA,MAAM,aAAa,IAAgB,OAA6B;AAC9D,UAAM,eAAe,KAAK;AAI1B,QAAI,aAA4B;AAKhC,UAAM,aAAa,OACjB,QAC2B;AAC3B,aAAO,eAAe,MAAM,yCAAyC;AACrE,YAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;AAI3D,UACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;AACA,YAAI,MAAM,mBAAmB,QAAW;AACtC,iBAAO,IAAI,QAAQ,YAAY,+BAA+B;AAC9D,eAAK;AAAA,YACH,qBAAqB,MAAM,YAAY,oCACF,IAAI,SAAS;AAAA,YAClD;AAAA,UAAA;AAAA,QAEJ;AACA,qBAAa,QAAQ,aAAa;AAClC,eAAO;AAAA,MACT;AAEA,YAAM,EAAC,OAAO,QAAQ,GAAA,IAAM,uBAAuB,aAAa;AAChE,YAAM,KAAK,qBAAqB;AAAA,QAC9B;AAAA,QACA,OAAO,OAAO,KAAK,IAAI;AAAA,MAAA,CACxB;AAED,WAAK,aAAa,KAAK;AAAA,QACrB;AAAA,QACA,EAAC,KAAK,SAAS,MAAM,KAAK,aAAa,SAAS,KAAA;AAAA,QAChD,EAAC,iBAAiB,GAAA;AAAA,MAAE,CACrB;AACD,aAAQ,aAAa;AAAA,IACvB;AAEA,UAAM,WAAW,MAAM;AACrB,UAAI,YAAY;AACd,aAAK,aAAa,KAAK;AAAA,UACrB;AAAA,UACA,EAAC,KAAK,SAAA;AAAA,UACN,EAAC,WAAW,WAAA;AAAA,QAAU,CACvB;AACD,qBAAa,QAAQ,UAAU;AAAA,MACjC;AACA,mBAAa;AAAA,IACf;AAEA,qBAAiB,OAAO,KAAK,kBAAkB,MAAM,OAAO,GAAG;AAG7D,YAAM,sBACJ,IAAI,QAAQ,wBACZ,KAAK,qBAAqB,IAAI,IAAI,SAAS;AAG7C,UACE,eACC,aAAa,YAAA,IAAgB,KAAK,sBACnC;AACA,iBAAA;AAAA,MACF;AAEA,6BAAwB,MAAM;AAE9B,UACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GACvC;AACA,cAAM,IAAI,mBAAmB,MAAM,OAAO;AAAA,MAC5C;AAGA,WAAK,eAAe,MAAM,WAAW,GAAG,OAAO,MAAM;AACnD,WAAG;AAAA,UACD,6BAA6B,MAAM,cAAc;AAAA,UACjD,MAAM;AAAA,QAAA;AAER,aAAK,uBAAA;AACL;AAAA,MACF;AAIA,YAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;AAAA,IACvC;AAGA,kBAAc,SAAA;AACd,OAAG,QAAQ,0BAA0B,MAAM,kBAAkB,EAAE;AAAA,EACjE;AAAA,EAEA,oBAAoB,OAAgD;AAClE,UAAM,QAAQ,KAAK;AACnB,WAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;AAAA,EACN;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,qBAAqB,QAAiB,UAAiC;AACrE,UAAM,WAAW,KAAK;AACtB,QAAI,YAAY,cAAc,YAAY,WAAW;AACnD,eAAS,iBAAiB;AAC1B,WAAK,mBAAmB;AACxB,gBAAU,KAAK,IAAI,OAAO,uBAAuB,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEA,qBAAqB,QAAgB,KAAsB;AACzD,UAAM,SAAS,KAAK,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;AACpE,SAAK,IAAI,OAAO,YAAY,MAAM,KAAK,MAAM,IAAI,EAAC,UAAU,IAAA,CAAI;AAChE,SAAK,mBAAmB,IAAI,IAAI,OAAO,GAAG;AAAA,EAC5C;AAAA,EAEA,wBAAwB,QAAgB,IAAgB;AACtD,UAAM,MAAM,KAAK,mBAAmB,IAAI,EAAE;AAC1C,QAAI,KAAK;AACP,YAAM,SAAS,WAAW,uBAAuB,cAAc;AAC/D,WAAK,IAAI,OAAO,YAAY,MAAM,KAAK,MAAM,IAAI,EAAC,UAAU,IAAA,CAAI;AAChE,WAAK,mBAAmB,OAAO,EAAE;AAAA,IACnC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,SAAS,SAAoC;AAC3C,QAAI,QAAQ,CAAC,MAAM,SAAS;AAC1B,WAAK,sBAAsB,QAAQ,CAAC,EAAE;AACtC;AAAA,IACF;AACA,QAAI,QAAQ,CAAC,MAAM,UAAU;AAC3B,WAAK,sBAAsB;AAC3B,WAAK,wBAAwB,QAAQ,CAAC,CAAC;AAGvC,WAAK,uBAAA;AACL;AAAA,IACF;AACA,QAAI,QAAQ,CAAC,MAAM,UAAU;AAC3B,WAAK,wBAAwB,QAAQ,CAAC,CAAC;AACvC;AAAA,IACF;AACA,QAAI,QAAQ,CAAC,MAAM,QAAQ;AACzB;AAAA,IACF;AACA,UAAM,SAAS,QAAQ,CAAC;AACxB,UAAM,EAAC,QAAO;AACd,YAAQ,KAAA;AAAA,MACN,KAAK,yBAAyB;AAC5B,cAAM,EAAC,OAAO,KAAK,SAAA,IAAY;AAC/B,cAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;AACzD,YAAI,iBAAiB;AACnB,eAAK,qBAAqB,KAAK;AAAA,YAC7B,GAAG;AAAA,YACH,OAAO,EAAC,GAAG,gBAAgB,OAAO,SAAA;AAAA,UAAQ,CAC3C;AACD,cAAI,KAAK,oBAAoB,KAAK,GAAG;AACnC,iBAAK,qBAAqB,uBAAuB;AAAA,UACnD;AAAA,QACF;AACA;AAAA,MACF;AAAA,MACA,KAAK,gBAAgB;AACnB,cAAM;AAAA,UACJ,MAAM,EAAC,QAAQ,KAAA;AAAA,UACf,WAAW;AAAA,UACX;AAAA,QAAA,IACE;AAEJ,YAAI,UAAU;AACZ,eAAK,qBAAqB,KAAK;AAAA,YAC7B,OAAO,EAAC,QAAQ,MAAM,SAAA;AAAA,YACtB,SAAS;AAAA,UAAA,CACV;AAAA,QACH;AACA;AAAA,MACF;AAAA,MACA,KAAK,gBAAgB;AACnB,cAAM,EAAC,KAAK,KAAK,SAAA,IAAY;AAC7B,cAAM,kBAAkB,KAAK,mBAAmB,IAAI,GAAG;AACvD,YAAI,iBAAiB;AACnB,gBAAM,EAAC,QAAQ,KAAA,IAAQ;AACvB,eAAK,wBAAwB,KAAK,GAAG;AACrC,eAAK,qBAAqB,KAAK;AAAA,YAC7B,GAAG;AAAA,YACH,OAAO,EAAC,GAAG,gBAAgB,OAAO,QAAQ,KAAA;AAAA,UAAI,CAC/C;AACD,cAAI,KAAK,oBAAoB,GAAG,GAAG;AACjC,iBAAK,qBAAqB,eAAe;AAAA,UAC3C;AAAA,QACF;AACA;AAAA,MACF;AAAA,MACA,KAAK,cAAc;AACjB,cAAM,EAAC,OAAM;AACb,cAAM,kBAAkB,KAAK,mBAAmB,IAAI,EAAE;AACtD,YAAI,iBAAiB;AACnB,eAAK,wBAAwB,KAAK,EAAE;AACpC,cAAI,KAAK,oBAAoB,EAAE,GAAG;AAChC,iBAAK,qBAAqB,eAAe;AAAA,UAC3C;AAAA,QACF;AACA;AAAA,MACF;AAAA,MACA,KAAK,cAAc;AACjB,cAAM;AAAA,UACJ;AAAA,UACA,eAAe,WAAW;AAAA,UAC1B;AAAA,UACA;AAAA,QAAA,IACE;AACJ,YAAI,UAAU;AACZ,gBAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;AACzD,cAAI,CAAC,iBAAiB;AACpB,iBAAK,qBAAqB,KAAK;AAAA,cAC7B,OAAO,EAAC,GAAG,OAAO,SAAA;AAAA,cAClB,SAAS,EAAC,CAAC,OAAO,IAAI,GAAG,SAAA;AAAA,YAAQ,CAClC;AAAA,UACH,OAAO;AACL,iBAAK,qBAAqB,KAAK;AAAA,cAC7B,GAAG;AAAA,cACH,OAAO,EAAC,GAAG,gBAAgB,OAAO,SAAA;AAAA,cAClC,SAAS;AAAA,gBACP,GAAG,gBAAgB;AAAA,gBACnB,CAAC,OAAO,IAAI,GAAG;AAAA,cAAA;AAAA,YACjB,CACD;AAAA,UAIH;AAAA,QACF;AACA;AAAA,MACF;AAAA,MACA,KAAK,iBAAiB;AACpB,cAAM;AAAA,UACJ;AAAA,UACA,KAAK,EAAC,MAAM,QAAA;AAAA,UACZ,KAAK,EAAC,MAAM,QAAA;AAAA,QAAO,IACjB;AACJ,YAAI,YAAY,SAAS;AACvB,gBAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;AACzD,cAAI,mBAAmB,WAAW,gBAAgB,SAAS;AACzD,kBAAM,EAAC,CAAC,OAAO,GAAG,SAAS,GAAG,UAAA,IAAa,gBAAgB;AAC3D,iBAAK,qBAAqB,KAAK;AAAA,cAC7B,GAAG;AAAA,cACH,SAAS,EAAC,GAAG,WAAW,CAAC,OAAO,GAAG,QAAA;AAAA,YAAO,CAC3C;AACD,kBAAM,WAAW,KAAK,oBAAoB,KAAK;AAC/C,gBAAI,YAAY,WAAW,SAAS,QAAQ,SAAS;AACnD,mBAAK,qBAAqB,gBAAgB;AAAA,YAC5C;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,MACA,KAAK,eAAe;AAClB,cAAM,EAAC,OAAO,OAAA,IAAU;AACxB,cAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;AACzD,YAAI,mBAAmB,UAAU,gBAAgB,SAAS;AACxD,gBAAM,EAAC,CAAC,MAAM,GAAG,WAAW,GAAG,UAAA,IAAa,gBAAgB;AAC5D,cAAI,OAAO,KAAK,SAAS,EAAE,WAAW,GAAG;AACvC,iBAAK,wBAAwB,KAAK,KAAK;AAAA,UACzC,OAAO;AACL,iBAAK,qBAAqB,KAAK;AAAA,cAC7B,GAAG;AAAA,cACH,SAAS;AAAA,YAAA,CACV;AAAA,UACH;AACA,gBAAM,WAAW,KAAK,oBAAoB,KAAK;AAC/C,cAAI,YAAY,UAAU,SAAS,QAAQ,SAAS;AAClD,iBAAK,qBAAqB,gBAAgB;AAAA,UAC5C;AAAA,QACF;AACA;AAAA,MACF;AAAA,MACA,KAAK,UAAU;AACb,cAAM,EAAC,UAAU,KAAK,KAAK,QAAO;AAClC,cAAM,WAAW,KAAK,oBAAoB,QAAQ;AAClD,cAAM,cAAc,KAAK,KAAK,qBAAqB,aAAa;AAChE,YAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ,MAAM;AAQpD,qBAAW,OAAO,OAAO;AAAA,YACvB,SAAS,QAAQ,MAAM,SAAS;AAAA,UAAA,GAC/B;AACD,gBAAI,IAAI,GAAG,MAAM,IAAI,GAAG,GAAG;AACzB,uBAAS,eAAe;AACxB,mBAAK,IAAI;AAAA,gBACP,gCAAgC,GAAG,qCACC,SAAS,YAAY;AAAA,cAAA;AAE3D;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,MACA,KAAK,sBAAsB;AACzB,cAAM,EAAC,UAAU,QAAA,IAAW;AAC5B,cAAM,kBAAkB,KAAK,mBAAmB,IAAI,QAAQ;AAC5D;AAAA,UACE;AAAA,UACA,MAAM,yCAAyC,UAAU,MAAM,CAAC;AAAA,QAAA;AAElE,cAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,EAAE;AAAA,UACxD,CAAC,CAAC,GAAG,MACH,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG;AAAA,QAAA;AAEnE,YAAI,UAAU,WAAW,GAAG;AAC1B,eAAK,wBAAwB,KAAK,QAAQ;AAAA,QAC5C,OAAO;AACL,eAAK,qBAAqB,KAAK;AAAA,YAC7B,GAAG;AAAA,YACH,SAAS,OAAO,YAAY,SAAS;AAAA,UAAA,CACtC;AAAA,QACH;AAGA,aAAK,qBAAA;AACL;AAAA,MACF;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEA,SAAe;AACb,SAAK,qBAAqB,wBAAwB;AAClD,iBAAa,KAAK,mBAAmB;AAAA,EACvC;AACF;AAEA,MAAe,4BAA4B,MAAM;AAAA,EAC/C,YAAY,IAAqB,KAAa,OAAiB;AAC7D;AAAA,MACE,mBAAmB,GAAG,MAAM,MAAM,IAAI,GAAG,MAAM,IAAI,IAC7C,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,CAAC,MAAM,GAAG;AAAA,MAChD,EAAC,MAAA;AAAA,IAAK;AAAA,EAEV;AACF;AAqBA,MAAM,2BAA2B,oBAAoB;AAAA,EAC1C,OAAO;AAAA,EAEhB,YAAY,IAAqB,OAAiB;AAChD,UAAM,IAAI,IAAI,GAAG,MAAM,IAAI,8BAA8B,KAAK;AAAA,EAChE;AACF;AAQO,MAAM,mCAAmC,oBAAoB;AAAA,EACzD,OAAO;AAAA,EAEhB,YAAY,IAAqB,KAAa,OAAiB;AAC7D,UAAM,IAAI,KAAK,KAAK;AAAA,EACtB;AACF;"}
+{"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<MessageBackfill | BackfillCompleted>;\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  canceledReason?: string | undefined;\n  minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {request, minWatermark: ''};\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    const log = this.#retryDelayMs === this.#maxBackoffMs ? 'error' : 'warn';\n    this.#lc[log]?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.canceledReason ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (state.canceledReason === undefined) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      const tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n      }\n      backfillTx = null;\n    };\n\n    for await (const msg of this.#backfillStreamer(state.request)) {\n      // Before sending `backfill-completed`, the main replication stream\n      // may need to catch up.\n      const mustWaitBeforeFlush =\n        msg.tag === 'backfill-completed' &&\n        this.#changeStreamReached(lc, msg.watermark);\n\n      // If necessary, yield the reservation to the main stream.\n      if (\n        backfillTx &&\n        (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush)\n      ) {\n        commitTx();\n      }\n\n      mustWaitBeforeFlush && (await mustWaitBeforeFlush);\n\n      if (\n        msg.tag === 'backfill' &&\n        msg.rowValues.length > 0 &&\n        msg.relation.rowKey.columns.length === 0\n      ) {\n        throw new MissingRowKeyError(state.request);\n      }\n\n      // Reserve the changeStreamer if not in a transaction.\n      if ((backfillTx ??= await beginTxFor(msg)) === null) {\n        lc.info?.(\n          `backfill stream canceled: ${state.canceledReason}`,\n          state.request,\n        );\n        this.#checkAndStartBackfill(); // start the next backfill if present\n        return; // this backfill is canceled\n      }\n\n      // `await` to allow the change streamer to exert back pressure\n      // on backfills.\n      await changeStream.push(['data', msg]);\n    }\n\n    // Flush any final tx and release the stream.\n    backfillTx && commitTx();\n    lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.canceledReason = reason;\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: backfill,\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: backfill},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: backfill,\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"mappings":";;;;;;;;;AAyBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;AAC5C,QAAO,GAAG,OAAO,GAAG;;AAatB,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,SACD;CACD;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,EAAE;;CAGlE,sBAAqC;CAErC,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,mBAAmB;AAC1D,QAAA,iBAAuB;AACvB,QAAA,mBAAyB;AACzB,QAAA,aAAmB;AACnB,QAAA,eAAqB;AACrB,QAAA,eAAqB;AACrB,QAAA,eAAqB;;CAGvB,IAAI,eAAuB,iBAAoC;AAC7D,QAAA,GAAS,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,iBAAgB,CAC5B;AACD,QAAA,sBAA4B;AAC5B,kBAAgB,SAAQ,QACtB,MAAA,oBAA0B,mBAAmB,IAAI,CAClD;AACD,QAAA,uBAA6B;;CAG/B,wBAAwB,EAAC,aAAiC;AAIxD,OAAK,MAAA,uBAA6B,MAAM,WAAW;AACjD,SAAA,sBAA4B;AAC5B,QAAK,IAAI,IAAI,MAAA,yBAA+B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,MAAA,yBAA+B;AAChD,QAAI,aAAa,SAAS,WAAW;AACnC,cAAS,SAAS;AAClB,WAAA,yBAA+B,OAAO,GAAG,EAAE;;;;;CAMnD,qBACE,IACA,WACsB;AACtB,OAAK,MAAA,uBAA6B,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,UAAU;AAC9C,SAAA,yBAA+B,KAAK;IAAC;IAAW;IAAQ,CAAC;AACzD,MAAG,OACD,iCAAiC,MAAA,oBAA0B,aAAa,YACzE;AACD,UAAO;;AAET,SAAO;;CAGT;CACA;CACA;CACA;CAEA,yBAAyB;AACvB,MACE,CAAC,MAAA,sBACD,CAAC,MAAA,mBACD,MAAA,kBAAwB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,MAAA,kBAAwB,QAAQ,CAAC;GACxD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,EAAE;GAC5D,MAAM,QAAQ;IAAC;IAAS,cAAc;IAAG;GACzC,MAAM,KAAK,MAAA,GAAS,YAAY,SAAS,QAAQ,MAAM,KAAK;AAE5D,SAAA,kBAAwB;AACnB,SAAA,YAAkB,IAAI,MAAM,CAC9B,WAAW;AACV,UAAA,oBAA0B,mBAAmB,MAAM;AACnD,UAAA,eAAqB,MAAA;KACrB,CAGD,OAAM,MAAK;AACV,UAAA,oBAA0B,OAAO,EAAE,EAAE,MAAM;AAC3C,UAAA,yBAA+B,EAAE;KACjC;;;CAIR,0BAA0B,GAAY;EACpC,MAAM,MAAM,MAAA,iBAAuB,MAAA,eAAqB,UAAU;AAClE,QAAA,GAAS,OACP,uCAAuC,MAAA,aAAmB,MAC1D,EACD;AACD,QAAA,qBAA2B,iBAAiB;AAC1C,SAAA,qBAA2B,KAAA;AAC3B,SAAA,uBAA6B;KAC5B,MAAA,aAAmB;AAEtB,QAAA,eAAqB,KAAK,IAAI,MAAA,eAAqB,GAAG,MAAA,aAAmB;;CAG3E,OAAA,YAAmB,IAAgB,OAA6B;EAC9D,MAAM,eAAe,MAAA;EAIrB,IAAI,aAA4B;;;;EAKhC,MAAM,aAAa,OACjB,QAC2B;AAC3B,UAAO,eAAe,MAAM,0CAA0C;GACtE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,WAAW;AAI5D,OACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;AACA,QAAI,MAAM,mBAAmB,KAAA,GAAW;AACtC,YAAO,IAAI,QAAQ,YAAY,gCAAgC;AAC/D,WAAA,oBACE,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,MACD;;AAEH,iBAAa,QAAQ,cAAc;AACnC,WAAO;;GAGT,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,cAAc;GACjE,MAAM,KAAK,qBAAqB;IAC9B;IACA,OAAO,OAAO,MAAM,GAAG;IACxB,CAAC;AAEG,gBAAa,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,MAAA;KAAkB,SAAS;KAAK;IACrD,EAAC,iBAAiB,IAAG;IACtB,CAAC;AACF,UAAQ,aAAa;;EAGvB,MAAM,iBAAiB;AACrB,OAAI,YAAY;AACT,iBAAa,KAAK;KACrB;KACA,EAAC,KAAK,UAAS;KACf,EAAC,WAAW,YAAW;KACxB,CAAC;AACF,iBAAa,QAAQ,WAAW;;AAElC,gBAAa;;AAGf,aAAW,MAAM,OAAO,MAAA,iBAAuB,MAAM,QAAQ,EAAE;GAG7D,MAAM,sBACJ,IAAI,QAAQ,wBACZ,MAAA,oBAA0B,IAAI,IAAI,UAAU;AAG9C,OACE,eACC,aAAa,aAAa,GAAG,KAAK,qBAEnC,WAAU;AAGZ,0BAAwB,MAAM;AAE9B,OACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,EAEvC,OAAM,IAAI,mBAAmB,MAAM,QAAQ;AAI7C,QAAK,eAAe,MAAM,WAAW,IAAI,MAAM,MAAM;AACnD,OAAG,OACD,6BAA6B,MAAM,kBACnC,MAAM,QACP;AACD,UAAA,uBAA6B;AAC7B;;AAKF,SAAM,aAAa,KAAK,CAAC,QAAQ,IAAI,CAAC;;AAIxC,gBAAc,UAAU;AACxB,KAAG,QAAQ,0BAA0B,MAAM,kBAAkB,GAAG;;CAGlE,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,MAAA;AACd,SAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;;;;;;;;CASN,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,MAAA;AACjB,MAAI,YAAY,cAAc,YAAY,WAAW;AACnD,YAAS,iBAAiB;AAC1B,SAAA,kBAAwB;AACxB,aAAU,MAAA,GAAS,OAAO,uBAAuB,OAAO;;;CAI5D,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,MAAA,kBAAwB,IAAI,IAAI,MAAM,GAAG,YAAY;AACpE,QAAA,GAAS,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,KAAI,CAAC;AACjE,QAAA,kBAAwB,IAAI,IAAI,OAAO,IAAI;;CAG7C,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,MAAA,kBAAwB,IAAI,GAAG;AAC3C,MAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;AAC/D,SAAA,GAAS,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,KAAI,CAAC;AACjE,SAAA,kBAAwB,OAAO,GAAG;;;;;;;CAQtC,SAAS,SAAoC;AAC3C,MAAI,QAAQ,OAAO,SAAS;AAC1B,SAAA,qBAA2B,QAAQ,GAAG;AACtC;;AAEF,MAAI,QAAQ,OAAO,UAAU;AAC3B,SAAA,qBAA2B;AAC3B,SAAA,uBAA6B,QAAQ,GAAG;AAGxC,SAAA,uBAA6B;AAC7B;;AAEF,MAAI,QAAQ,OAAO,UAAU;AAC3B,SAAA,uBAA6B,QAAQ,GAAG;AACxC;;AAEF,MAAI,QAAQ,OAAO,OACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;AACd,UAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,QAAI,iBAAiB;AACnB,WAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAS;MAC5C,CAAC;AACF,SAAI,MAAA,mBAAyB,MAAM,CACjC,OAAA,oBAA0B,wBAAwB;;AAGtD;;GAEF,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;AAEJ,QAAI,SACF,OAAA,oBAA0B,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;MAAS;KAC/B,SAAS;KACV,CAAC;AAEJ;;GAEF,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,IAAI;AACxD,QAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;AACvB,WAAA,uBAA6B,KAAK,IAAI;AACtC,WAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;OAAK;MAChD,CAAC;AACF,SAAI,MAAA,mBAAyB,IAAI,CAC/B,OAAA,oBAA0B,gBAAgB;;AAG9C;;GAEF,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;AAEb,QADwB,MAAA,kBAAwB,IAAI,GAAG,EAClC;AACnB,WAAA,uBAA6B,KAAK,GAAG;AACrC,SAAI,MAAA,mBAAyB,GAAG,CAC9B,OAAA,oBAA0B,gBAAgB;;AAG9C;;GAEF,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;AACJ,QAAI,UAAU;KACZ,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,SAAI,CAAC,gBACH,OAAA,oBAA0B,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;OAAS;MAC3B,SAAS,GAAE,OAAO,OAAO,UAAS;MACnC,CAAC;SAEF,OAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAS;MAC3C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO;OAChB;MACF,CAAC;;AAMN;;GAEF,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;AACJ,QAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,SAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;AAC3D,YAAA,oBAA0B,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;QAAQ;OAC5C,CAAC;MACF,MAAM,WAAW,MAAA,mBAAyB,MAAM;AAChD,UAAI,YAAY,WAAW,SAAS,QAAQ,QAC1C,OAAA,oBAA0B,iBAAiB;;;AAIjD;;GAEF,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,QAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;AAC5D,SAAI,OAAO,KAAK,UAAU,CAAC,WAAW,EACpC,OAAA,uBAA6B,KAAK,MAAM;SAExC,OAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,SAAS;MACV,CAAC;KAEJ,MAAM,WAAW,MAAA,mBAAyB,MAAM;AAChD,SAAI,YAAY,UAAU,SAAS,QAAQ,QACzC,OAAA,oBAA0B,iBAAiB;;AAG/C;;GAEF,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,MAAA,mBAAyB,SAAS;IACnD,MAAM,cAAc,KAAK,MAAA,oBAA0B,cAAc;AACjE,QAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,OACjC,CACC,KAAI,IAAI,SAAS,IAAI,MAAM;AACzB,eAAS,eAAe;AACxB,YAAA,GAAS,OACP,gCAAgC,IAAI,oCACA,SAAS,eAC9C;AACD;;;AAIN;;GAEF,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,SAAS;AAC7D,WACE,uBACM,yCAAyC,UAAU,OAAO,GACjE;IACD,MAAM,YAAY,OAAO,QAAQ,gBAAgB,QAAQ,CAAC,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,IAAI,IAAI,SAAS,OAAO,QAAQ,SAAS,IAAI,EACnE;AACD,QAAI,UAAU,WAAW,EACvB,OAAA,uBAA6B,KAAK,SAAS;QAE3C,OAAA,oBAA0B,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,UAAU;KACvC,CAAC;AAIJ,UAAA,qBAA2B;AAC3B;;;;CAKN,SAAe;AACb,QAAA,oBAA0B,yBAAyB;AACnD,eAAa,MAAA,mBAAyB;;;AAI1C,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;AAC7D,QACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,CAAC,KAAK,OAC7C,EAAC,OAAM,CACR;;;;;;;;;;;;;;;;;;;;;;AAuBL,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;AAChD,QAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,MAAM;;;;;;;;;AAUnE,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;AAC7D,QAAM,IAAI,KAAK,MAAM"}

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js

@@ -1,117 +1,114 @@
-import { resolver } from "@rocicorp/resolver";
 import { assert } from "../../../../../shared/src/asserts.js";
 import { Subscription } from "../../../types/subscription.js";
-class ChangeStreamMultiplexer {
-  #lc;
-  #sub;
-  #producers = [];
-  #listeners = [];
-  /**
-   * The `#lastWatermark` tracks the watermark of the last transaction
-   * committed to stream. It is set to null when a producer has reserved
-   * the stream, and set to the new watermark when the producer releases
-   * the reservation.
-   */
-  #lastWatermark;
-  /**
-   * Tracks the queue of producers waiting for a reservation.
-   */
-  #waiters = [];
-  constructor(lc, lastWatermark) {
-    this.#lc = lc;
-    this.#sub = Subscription.create({
-      cleanup: () => this.#producers.forEach((p) => p.cancel())
-    });
-    this.#lastWatermark = lastWatermark;
-  }
-  addProducers(...p) {
-    this.#producers.push(...p);
-    return this;
-  }
-  addListeners(...l) {
-    this.#listeners.push(...l);
-    return this;
-  }
-  /**
-   * Called by a producer to "reserve" the exclusive right to push data
-   * changes to the stream. If the stream is currently reserved,
-   * the resulting Promise must be awaited.
-   *
-   * The producer must then {@link release()} the reservation when it sees
-   * fit.
-   *
-   * @param producer The name of the producer, purely for debugging output
-   */
-  reserve(producer) {
-    if (this.#lastWatermark !== null) {
-      const lastWatermark = this.#lastWatermark;
-      this.#lastWatermark = null;
-      return lastWatermark;
-    }
-    const startTime = performance.now();
-    const { promise, resolve: grantReservation } = resolver();
-    this.#waiters.push({ producer, startTime, grantReservation });
-    return promise;
-  }
-  /**
-   * If there are producers currently awaiting a reservation, returns
-   * the duration (in milliseconds) of the oldest reservation request.
-   * Returns a negative number if there are no waiters.
-   */
-  waiterDelay() {
-    if (this.#waiters.length === 0) {
-      return -1;
-    }
-    return performance.now() - this.#waiters[0].startTime;
-  }
-  /**
-   * Called by a producer to release its reservation after committing its
-   * last transaction.
-   */
-  release(newWatermark) {
-    const waiter = this.#waiters.shift();
-    if (!waiter) {
-      this.#lastWatermark = newWatermark;
-    } else {
-      const { producer, startTime, grantReservation } = waiter;
-      grantReservation(newWatermark);
-      const elapsed = performance.now() - startTime;
-      this.#lc.info?.(
-        `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`
-      );
-    }
-  }
-  /**
-   * `pushStatus()` can be called without a reservation, as it
-   * does not constitute a data change and can appear anywhere
-   * in the stream.
-   */
-  pushStatus(message) {
-    this.#listeners.forEach((l) => l.onChange(message));
-    if (message[1].ack) {
-      this.#sub.push(message);
-    }
-  }
-  /**
-   * `push()` must only be called by a producer after it has
-   * {@link reserve}d the stream.
-   */
-  push(message) {
-    assert(
-      this.#lastWatermark === null,
-      `push() called without reserve()-ing the stream`
-    );
-    this.#listeners.forEach((l) => l.onChange(message));
-    return this.#sub.push(message).result;
-  }
-  fail(err) {
-    this.#sub.fail(err);
-  }
-  asSource() {
-    return this.#sub;
-  }
-}
-export {
-  ChangeStreamMultiplexer
+import { resolver } from "@rocicorp/resolver";
+//#region ../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts
+/**
+* Facilitates cooperative multiplexing of transactions from
+* multiple producers into a single change stream.
+*/
+var ChangeStreamMultiplexer = class {
+	#lc;
+	#sub;
+	#producers = [];
+	#listeners = [];
+	/**
+	* The `#lastWatermark` tracks the watermark of the last transaction
+	* committed to stream. It is set to null when a producer has reserved
+	* the stream, and set to the new watermark when the producer releases
+	* the reservation.
+	*/
+	#lastWatermark;
+	/**
+	* Tracks the queue of producers waiting for a reservation.
+	*/
+	#waiters = [];
+	constructor(lc, lastWatermark) {
+		this.#lc = lc;
+		this.#sub = Subscription.create({ cleanup: () => this.#producers.forEach((p) => p.cancel()) });
+		this.#lastWatermark = lastWatermark;
+	}
+	addProducers(...p) {
+		this.#producers.push(...p);
+		return this;
+	}
+	addListeners(...l) {
+		this.#listeners.push(...l);
+		return this;
+	}
+	/**
+	* Called by a producer to "reserve" the exclusive right to push data
+	* changes to the stream. If the stream is currently reserved,
+	* the resulting Promise must be awaited.
+	*
+	* The producer must then {@link release()} the reservation when it sees
+	* fit.
+	*
+	* @param producer The name of the producer, purely for debugging output
+	*/
+	reserve(producer) {
+		if (this.#lastWatermark !== null) {
+			const lastWatermark = this.#lastWatermark;
+			this.#lastWatermark = null;
+			return lastWatermark;
+		}
+		const startTime = performance.now();
+		const { promise, resolve: grantReservation } = resolver();
+		this.#waiters.push({
+			producer,
+			startTime,
+			grantReservation
+		});
+		return promise;
+	}
+	/**
+	* If there are producers currently awaiting a reservation, returns
+	* the duration (in milliseconds) of the oldest reservation request.
+	* Returns a negative number if there are no waiters.
+	*/
+	waiterDelay() {
+		if (this.#waiters.length === 0) return -1;
+		return performance.now() - this.#waiters[0].startTime;
+	}
+	/**
+	* Called by a producer to release its reservation after committing its
+	* last transaction.
+	*/
+	release(newWatermark) {
+		const waiter = this.#waiters.shift();
+		if (!waiter) this.#lastWatermark = newWatermark;
+		else {
+			const { producer, startTime, grantReservation } = waiter;
+			grantReservation(newWatermark);
+			const elapsed = performance.now() - startTime;
+			this.#lc.info?.(`${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`);
+		}
+	}
+	/**
+	* `pushStatus()` can be called without a reservation, as it
+	* does not constitute a data change and can appear anywhere
+	* in the stream.
+	*/
+	pushStatus(message) {
+		this.#listeners.forEach((l) => l.onChange(message));
+		if (message[1].ack) this.#sub.push(message);
+	}
+	/**
+	* `push()` must only be called by a producer after it has
+	* {@link reserve}d the stream.
+	*/
+	push(message) {
+		assert(this.#lastWatermark === null, `push() called without reserve()-ing the stream`);
+		this.#listeners.forEach((l) => l.onChange(message));
+		return this.#sub.push(message).result;
+	}
+	fail(err) {
+		this.#sub.fail(err);
+	}
+	asSource() {
+		return this.#sub;
+	}
 };
-//# sourceMappingURL=change-stream-multiplexer.js.map
+//#endregion
+export { ChangeStreamMultiplexer };
+
+//# sourceMappingURL=change-stream-multiplexer.js.map

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-stream-multiplexer.js","sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Source} from '../../../types/streams.ts';\nimport {Subscription} from '../../../types/subscription.ts';\nimport type {\n  ChangeStreamMessage,\n  DownstreamStatusMessage,\n} from '../protocol/current.ts';\n\nexport type Cancelable = {\n  cancel(): void;\n};\n\nexport type Listener = {\n  onChange(change: ChangeStreamMessage): void;\n};\n\ntype Waiter = {\n  producer: string;\n  startTime: number;\n  grantReservation: (watermark: string) => void;\n};\n\n/**\n * Facilitates cooperative multiplexing of transactions from\n * multiple producers into a single change stream.\n */\nexport class ChangeStreamMultiplexer {\n  readonly #lc: LogContext;\n  readonly #sub: Subscription<ChangeStreamMessage>;\n  readonly #producers: Cancelable[] = [];\n  readonly #listeners: Listener[] = [];\n\n  /**\n   * The `#lastWatermark` tracks the watermark of the last transaction\n   * committed to stream. It is set to null when a producer has reserved\n   * the stream, and set to the new watermark when the producer releases\n   * the reservation.\n   */\n  #lastWatermark: string | null;\n\n  /**\n   * Tracks the queue of producers waiting for a reservation.\n   */\n  readonly #waiters: Waiter[] = [];\n\n  constructor(lc: LogContext, lastWatermark: string) {\n    this.#lc = lc;\n    this.#sub = Subscription.create<ChangeStreamMessage>({\n      cleanup: () => this.#producers.forEach(p => p.cancel()),\n    });\n    this.#lastWatermark = lastWatermark;\n  }\n\n  addProducers(...p: Cancelable[]): this {\n    this.#producers.push(...p);\n    return this;\n  }\n\n  addListeners(...l: Listener[]): this {\n    this.#listeners.push(...l);\n    return this;\n  }\n\n  /**\n   * Called by a producer to \"reserve\" the exclusive right to push data\n   * changes to the stream. If the stream is currently reserved,\n   * the resulting Promise must be awaited.\n   *\n   * The producer must then {@link release()} the reservation when it sees\n   * fit.\n   *\n   * @param producer The name of the producer, purely for debugging output\n   */\n  reserve(producer: string): string | Promise<string> {\n    if (this.#lastWatermark !== null) {\n      // If the stream is not reserved, reserve it and return the\n      // watermark.\n      const lastWatermark = this.#lastWatermark;\n      this.#lastWatermark = null;\n      return lastWatermark;\n    }\n\n    // Otherwise, wait for the current reservation to be released.\n    const startTime = performance.now();\n    const {promise, resolve: grantReservation} = resolver<string>();\n    this.#waiters.push({producer, startTime, grantReservation});\n\n    return promise;\n  }\n\n  /**\n   * If there are producers currently awaiting a reservation, returns\n   * the duration (in milliseconds) of the oldest reservation request.\n   * Returns a negative number if there are no waiters.\n   */\n  waiterDelay(): number {\n    if (this.#waiters.length === 0) {\n      return -1;\n    }\n    return performance.now() - this.#waiters[0].startTime;\n  }\n\n  /**\n   * Called by a producer to release its reservation after committing its\n   * last transaction.\n   */\n  release(newWatermark: string) {\n    const waiter = this.#waiters.shift();\n    if (!waiter) {\n      this.#lastWatermark = newWatermark;\n    } else {\n      const {producer, startTime, grantReservation} = waiter;\n      grantReservation(newWatermark);\n      const elapsed = performance.now() - startTime;\n      this.#lc.info?.(\n        `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`,\n      );\n    }\n  }\n\n  /**\n   * `pushStatus()` can be called without a reservation, as it\n   * does not constitute a data change and can appear anywhere\n   * in the stream.\n   */\n  pushStatus(message: DownstreamStatusMessage) {\n    // Let listeners know about all status messages.\n    this.#listeners.forEach(l => l.onChange(message));\n    // The ChangeStreamer only cares about status messages requiring an ack.\n    // To reduce churn, avoid sending other status messages.\n    if (message[1].ack) {\n      this.#sub.push(message);\n    }\n  }\n\n  /**\n   * `push()` must only be called by a producer after it has\n   * {@link reserve}d the stream.\n   */\n  push(message: ChangeStreamMessage): Promise<unknown> {\n    assert(\n      this.#lastWatermark === null,\n      `push() called without reserve()-ing the stream`,\n    );\n    this.#listeners.forEach(l => l.onChange(message));\n    return this.#sub.push(message).result;\n  }\n\n  fail(err: Error) {\n    this.#sub.fail(err);\n  }\n\n  asSource(): Source<ChangeStreamMessage> {\n    return this.#sub;\n  }\n}\n"],"names":[],"mappings":";;;AA4BO,MAAM,wBAAwB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA,aAA2B,CAAA;AAAA,EAC3B,aAAyB,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQlC;AAAA;AAAA;AAAA;AAAA,EAKS,WAAqB,CAAA;AAAA,EAE9B,YAAY,IAAgB,eAAuB;AACjD,SAAK,MAAM;AACX,SAAK,OAAO,aAAa,OAA4B;AAAA,MACnD,SAAS,MAAM,KAAK,WAAW,QAAQ,CAAA,MAAK,EAAE,QAAQ;AAAA,IAAA,CACvD;AACD,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,gBAAgB,GAAuB;AACrC,SAAK,WAAW,KAAK,GAAG,CAAC;AACzB,WAAO;AAAA,EACT;AAAA,EAEA,gBAAgB,GAAqB;AACnC,SAAK,WAAW,KAAK,GAAG,CAAC;AACzB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,QAAQ,UAA4C;AAClD,QAAI,KAAK,mBAAmB,MAAM;AAGhC,YAAM,gBAAgB,KAAK;AAC3B,WAAK,iBAAiB;AACtB,aAAO;AAAA,IACT;AAGA,UAAM,YAAY,YAAY,IAAA;AAC9B,UAAM,EAAC,SAAS,SAAS,iBAAA,IAAoB,SAAA;AAC7C,SAAK,SAAS,KAAK,EAAC,UAAU,WAAW,kBAAiB;AAE1D,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,cAAsB;AACpB,QAAI,KAAK,SAAS,WAAW,GAAG;AAC9B,aAAO;AAAA,IACT;AACA,WAAO,YAAY,IAAA,IAAQ,KAAK,SAAS,CAAC,EAAE;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,QAAQ,cAAsB;AAC5B,UAAM,SAAS,KAAK,SAAS,MAAA;AAC7B,QAAI,CAAC,QAAQ;AACX,WAAK,iBAAiB;AAAA,IACxB,OAAO;AACL,YAAM,EAAC,UAAU,WAAW,iBAAA,IAAoB;AAChD,uBAAiB,YAAY;AAC7B,YAAM,UAAU,YAAY,IAAA,IAAQ;AACpC,WAAK,IAAI;AAAA,QACP,GAAG,QAAQ,WAAW,QAAQ,QAAQ,CAAC,CAAC;AAAA,MAAA;AAAA,IAE5C;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,WAAW,SAAkC;AAE3C,SAAK,WAAW,QAAQ,CAAA,MAAK,EAAE,SAAS,OAAO,CAAC;AAGhD,QAAI,QAAQ,CAAC,EAAE,KAAK;AAClB,WAAK,KAAK,KAAK,OAAO;AAAA,IACxB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,KAAK,SAAgD;AACnD;AAAA,MACE,KAAK,mBAAmB;AAAA,MACxB;AAAA,IAAA;AAEF,SAAK,WAAW,QAAQ,CAAA,MAAK,EAAE,SAAS,OAAO,CAAC;AAChD,WAAO,KAAK,KAAK,KAAK,OAAO,EAAE;AAAA,EACjC;AAAA,EAEA,KAAK,KAAY;AACf,SAAK,KAAK,KAAK,GAAG;AAAA,EACpB;AAAA,EAEA,WAAwC;AACtC,WAAO,KAAK;AAAA,EACd;AACF;"}
+{"version":3,"file":"change-stream-multiplexer.js","names":["#lc","#sub","#producers","#listeners","#waiters","#lastWatermark"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Source} from '../../../types/streams.ts';\nimport {Subscription} from '../../../types/subscription.ts';\nimport type {\n  ChangeStreamMessage,\n  DownstreamStatusMessage,\n} from '../protocol/current.ts';\n\nexport type Cancelable = {\n  cancel(): void;\n};\n\nexport type Listener = {\n  onChange(change: ChangeStreamMessage): void;\n};\n\ntype Waiter = {\n  producer: string;\n  startTime: number;\n  grantReservation: (watermark: string) => void;\n};\n\n/**\n * Facilitates cooperative multiplexing of transactions from\n * multiple producers into a single change stream.\n */\nexport class ChangeStreamMultiplexer {\n  readonly #lc: LogContext;\n  readonly #sub: Subscription<ChangeStreamMessage>;\n  readonly #producers: Cancelable[] = [];\n  readonly #listeners: Listener[] = [];\n\n  /**\n   * The `#lastWatermark` tracks the watermark of the last transaction\n   * committed to stream. It is set to null when a producer has reserved\n   * the stream, and set to the new watermark when the producer releases\n   * the reservation.\n   */\n  #lastWatermark: string | null;\n\n  /**\n   * Tracks the queue of producers waiting for a reservation.\n   */\n  readonly #waiters: Waiter[] = [];\n\n  constructor(lc: LogContext, lastWatermark: string) {\n    this.#lc = lc;\n    this.#sub = Subscription.create<ChangeStreamMessage>({\n      cleanup: () => this.#producers.forEach(p => p.cancel()),\n    });\n    this.#lastWatermark = lastWatermark;\n  }\n\n  addProducers(...p: Cancelable[]): this {\n    this.#producers.push(...p);\n    return this;\n  }\n\n  addListeners(...l: Listener[]): this {\n    this.#listeners.push(...l);\n    return this;\n  }\n\n  /**\n   * Called by a producer to \"reserve\" the exclusive right to push data\n   * changes to the stream. If the stream is currently reserved,\n   * the resulting Promise must be awaited.\n   *\n   * The producer must then {@link release()} the reservation when it sees\n   * fit.\n   *\n   * @param producer The name of the producer, purely for debugging output\n   */\n  reserve(producer: string): string | Promise<string> {\n    if (this.#lastWatermark !== null) {\n      // If the stream is not reserved, reserve it and return the\n      // watermark.\n      const lastWatermark = this.#lastWatermark;\n      this.#lastWatermark = null;\n      return lastWatermark;\n    }\n\n    // Otherwise, wait for the current reservation to be released.\n    const startTime = performance.now();\n    const {promise, resolve: grantReservation} = resolver<string>();\n    this.#waiters.push({producer, startTime, grantReservation});\n\n    return promise;\n  }\n\n  /**\n   * If there are producers currently awaiting a reservation, returns\n   * the duration (in milliseconds) of the oldest reservation request.\n   * Returns a negative number if there are no waiters.\n   */\n  waiterDelay(): number {\n    if (this.#waiters.length === 0) {\n      return -1;\n    }\n    return performance.now() - this.#waiters[0].startTime;\n  }\n\n  /**\n   * Called by a producer to release its reservation after committing its\n   * last transaction.\n   */\n  release(newWatermark: string) {\n    const waiter = this.#waiters.shift();\n    if (!waiter) {\n      this.#lastWatermark = newWatermark;\n    } else {\n      const {producer, startTime, grantReservation} = waiter;\n      grantReservation(newWatermark);\n      const elapsed = performance.now() - startTime;\n      this.#lc.info?.(\n        `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`,\n      );\n    }\n  }\n\n  /**\n   * `pushStatus()` can be called without a reservation, as it\n   * does not constitute a data change and can appear anywhere\n   * in the stream.\n   */\n  pushStatus(message: DownstreamStatusMessage) {\n    // Let listeners know about all status messages.\n    this.#listeners.forEach(l => l.onChange(message));\n    // The ChangeStreamer only cares about status messages requiring an ack.\n    // To reduce churn, avoid sending other status messages.\n    if (message[1].ack) {\n      this.#sub.push(message);\n    }\n  }\n\n  /**\n   * `push()` must only be called by a producer after it has\n   * {@link reserve}d the stream.\n   */\n  push(message: ChangeStreamMessage): Promise<unknown> {\n    assert(\n      this.#lastWatermark === null,\n      `push() called without reserve()-ing the stream`,\n    );\n    this.#listeners.forEach(l => l.onChange(message));\n    return this.#sub.push(message).result;\n  }\n\n  fail(err: Error) {\n    this.#sub.fail(err);\n  }\n\n  asSource(): Source<ChangeStreamMessage> {\n    return this.#sub;\n  }\n}\n"],"mappings":";;;;;;;;AA4BA,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA,aAAoC,EAAE;CACtC,aAAkC,EAAE;;;;;;;CAQpC;;;;CAKA,WAA8B,EAAE;CAEhC,YAAY,IAAgB,eAAuB;AACjD,QAAA,KAAW;AACX,QAAA,MAAY,aAAa,OAA4B,EACnD,eAAe,MAAA,UAAgB,SAAQ,MAAK,EAAE,QAAQ,CAAC,EACxD,CAAC;AACF,QAAA,gBAAsB;;CAGxB,aAAa,GAAG,GAAuB;AACrC,QAAA,UAAgB,KAAK,GAAG,EAAE;AAC1B,SAAO;;CAGT,aAAa,GAAG,GAAqB;AACnC,QAAA,UAAgB,KAAK,GAAG,EAAE;AAC1B,SAAO;;;;;;;;;;;;CAaT,QAAQ,UAA4C;AAClD,MAAI,MAAA,kBAAwB,MAAM;GAGhC,MAAM,gBAAgB,MAAA;AACtB,SAAA,gBAAsB;AACtB,UAAO;;EAIT,MAAM,YAAY,YAAY,KAAK;EACnC,MAAM,EAAC,SAAS,SAAS,qBAAoB,UAAkB;AAC/D,QAAA,QAAc,KAAK;GAAC;GAAU;GAAW;GAAiB,CAAC;AAE3D,SAAO;;;;;;;CAQT,cAAsB;AACpB,MAAI,MAAA,QAAc,WAAW,EAC3B,QAAO;AAET,SAAO,YAAY,KAAK,GAAG,MAAA,QAAc,GAAG;;;;;;CAO9C,QAAQ,cAAsB;EAC5B,MAAM,SAAS,MAAA,QAAc,OAAO;AACpC,MAAI,CAAC,OACH,OAAA,gBAAsB;OACjB;GACL,MAAM,EAAC,UAAU,WAAW,qBAAoB;AAChD,oBAAiB,aAAa;GAC9B,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,SAAA,GAAS,OACP,GAAG,SAAS,UAAU,QAAQ,QAAQ,EAAE,CAAC,4BAC1C;;;;;;;;CASL,WAAW,SAAkC;AAE3C,QAAA,UAAgB,SAAQ,MAAK,EAAE,SAAS,QAAQ,CAAC;AAGjD,MAAI,QAAQ,GAAG,IACb,OAAA,IAAU,KAAK,QAAQ;;;;;;CAQ3B,KAAK,SAAgD;AACnD,SACE,MAAA,kBAAwB,MACxB,iDACD;AACD,QAAA,UAAgB,SAAQ,MAAK,EAAE,SAAS,QAAQ,CAAC;AACjD,SAAO,MAAA,IAAU,KAAK,QAAQ,CAAC;;CAGjC,KAAK,KAAY;AACf,QAAA,IAAU,KAAK,IAAI;;CAGrB,WAAwC;AACtC,SAAO,MAAA"}