@rocicorp/zero 1.4.0-canary.4 → 1.4.0-canary.6

package/out/zero-cache/src/services/change-streamer/storer.js.map

@@ -1 +1 @@
-{"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#db","#replicaVersion","#onConsumed","#onFatal","#queue","#backPressureThresholdBytes","#statementTimeoutMs","#cdc","#approximateQueuedBytes","#running","#readyForMore","#stopped","#processQueue","#cancelQueueEntries","#startCatchup","#trackBackfillMetadata","#maybeReleaseBackPressure","#catchup","#upsertTableMetadataStmt","#upsertColumnBackfillStmt","#tx","#released"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {getHeapStatistics} from 'node:v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {\n  backfillRequestSchema,\n  isDataChange,\n  isSchemaChange,\n  type BackfillID,\n  type BackfillRequest,\n  type Change,\n  type DataChange,\n  type Identifier,\n  type SchemaChange,\n  type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {type Commit} from '../change-source/protocol/current/downstream.ts';\nimport type {\n  DownstreamStatusMessage,\n  UpstreamStatusMessage,\n} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type ChangeEntry} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n  AutoResetSignal,\n  markResetRequired,\n  type BackfillingColumn,\n  type ReplicationState,\n  type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n  subscriber: Subscriber;\n  mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n  | [\n      'change',\n      watermark: string,\n      json: string,\n      orig: Exclude<Change, DataChange> | null, // null for DataChanges\n    ]\n  | ['ready', callback: () => void]\n  | ['subscriber', SubscriberAndMode]\n  | DownstreamStatusMessage\n  | ['abort']\n  | 'stop';\n\ntype PendingTransaction = {\n  pool: TransactionPool;\n  preCommitWatermark: string;\n  pos: number;\n  startingReplicationState: Promise<ReplicationState>;\n  ack: boolean;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\nexport type TuningOptions = {\n  backPressureLimitHeapProportion: number;\n  statementTimeoutMs: number;\n};\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n *   `replicaVersion`, which is the version at which initial sync\n *   was performed, and any attempts to catchup from a different\n *   point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n *   a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n *   subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n  readonly id = 'storer';\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #taskID: string;\n  readonly #discoveryAddress: string;\n  readonly #discoveryProtocol: string;\n  readonly #db: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #onConsumed: (c: Commit | UpstreamStatusMessage) => void;\n  readonly #onFatal: (err: Error) => void;\n  readonly #queue = new Queue<QueueEntry>();\n  readonly #backPressureThresholdBytes: number;\n  readonly #statementTimeoutMs: number;\n\n  #approximateQueuedBytes = 0;\n  #running = false;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    db: PostgresDB,\n    replicaVersion: string,\n    onConsumed: (c: Commit | UpstreamStatusMessage) => void,\n    onFatal: (err: Error) => void,\n    {backPressureLimitHeapProportion, statementTimeoutMs}: TuningOptions,\n  ) {\n    this.#lc = lc.withContext('component', 'change-log');\n    this.#shard = shard;\n    this.#taskID = taskID;\n    this.#discoveryAddress = discoveryAddress;\n    this.#discoveryProtocol = discoveryProtocol;\n    this.#db = db;\n    this.#replicaVersion = replicaVersion;\n    this.#onConsumed = onConsumed;\n    this.#onFatal = onFatal;\n    this.#statementTimeoutMs = statementTimeoutMs;\n\n    const heapStats = getHeapStatistics();\n    this.#backPressureThresholdBytes =\n      (heapStats.heap_size_limit - heapStats.used_heap_size) *\n      backPressureLimitHeapProportion;\n\n    this.#lc.info?.(\n      `Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of ` +\n        `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) ` +\n        `to absorb upstream spikes`,\n      {heapStats},\n    );\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async assumeOwnership(purgeLock?: PurgeLock | null) {\n    const db = this.#db;\n    const owner = this.#taskID;\n    const ownerAddress = this.#discoveryAddress;\n    const ownerProtocol = this.#discoveryProtocol;\n    // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n    const addressWithProtocol =\n      ownerProtocol === 'ws'\n        ? ownerAddress\n        : `${ownerProtocol}://${ownerAddress}`;\n    this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n    const start = performance.now();\n    await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n    const elapsed = (performance.now() - start).toFixed(2);\n    this.#lc.info?.(\n      `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n    );\n\n    if (purgeLock) {\n      // Once ownership has been assumed, any initial purge-lock preventing the\n      // purging of change-log records can be released, as a change-streamer\n      // that was attempting to purge records will correspondingly abort on the\n      // ownership check.\n      void purgeLock.release();\n    }\n  }\n\n  async getStartStreamInitializationParameters(): Promise<{\n    lastWatermark: string;\n    backfillRequests: BackfillRequest[];\n  }> {\n    const [[{lastWatermark}], result] = await runTx(\n      this.#db,\n      sql => [\n        sql<{lastWatermark: string}[]>`\n        SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n        // Formats a BackfillRequest using json_object_agg() to construct the\n        // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n        // to make it optional and possibly `null`.\n        sql`\n        SELECT \n            json_build_object(\n              'schema', b.\"schema\",\n              'name', b.\"table\",\n              'metadata', t.\"metadata\"\n            ) as \"table\",\n            json_object_agg(b.\"column\", b.\"backfill\") \n              as \"columns\"\n          FROM ${this.#cdc('backfilling')} as b\n          LEFT JOIN ${this.#cdc('tableMetadata')} as t\n          ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n          GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n        `,\n      ],\n      {mode: Mode.READONLY},\n    );\n\n    return {\n      lastWatermark,\n      backfillRequests: v.parse(result, backfillRequestsSchema),\n    };\n  }\n\n  async getMinWatermarkForCatchup(): Promise<string | null> {\n    const [{minWatermark}] = await this.#db<{minWatermark: string | null}[]>\n    /*sql*/ `\n      SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n    return minWatermark;\n  }\n\n  purgeRecordsBefore(watermark: string): Promise<number> {\n    return runTx(this.#db, async sql => {\n      // This NOWAIT pre-check is an optimization to abort the transaction\n      // (and release associated resources) early.\n      await sql<{watermark: string}[]>`\n          SELECT watermark FROM ${this.#cdc('changeLog')}\n            ORDER BY watermark, pos LIMIT 1\n            FOR UPDATE NOWAIT\n        `;\n      // If the row is purge-locked by an incoming replication-manager, it\n      // will assume ownership of the change-log before releasing the lock.\n      // This DELETE blocks until the lock is released, allowing the change\n      // in ownership to be reliably detected (and the transaction aborted)\n      // in the subsequent check.\n      const [{deleted}] = await sql<{deleted: bigint}[]>`\n        WITH purged AS (\n          DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n            RETURNING watermark, pos\n        ) SELECT COUNT(*) as deleted FROM purged;`;\n\n      const [{owner}] = await sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')} FOR SHARE`;\n      if (owner !== this.#taskID) {\n        throw new AbortError(\n          `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n        );\n      }\n      return Number(deleted);\n    });\n  }\n\n  /**\n   * @returns The size of the serialized entry, for memory / I/O estimations.\n   */\n  store(entry: WatermarkedChange) {\n    const [watermark, [_tag, change]] = entry;\n    // Eagerly stringify the JSON object so that the memory usage can be\n    // more accurately measured (i.e. without an extra object traversal and\n    // ad hoc memory counting heuristics).\n    //\n    // This essentially moves the stringify() computation out of the pg client,\n    // which is instead configured to pass `string` objects directly as JSON\n    // strings for JSON-valued columns (see TypeOptions.sendStringAsJson).\n    const json = BigIntJSON.stringify(change);\n    this.#approximateQueuedBytes += json.length;\n\n    this.#queue.enqueue([\n      'change',\n      watermark,\n      json,\n      isDataChange(change) ? null : change, // drop DataChanges to save memory\n    ]);\n\n    return json.length;\n  }\n\n  abort() {\n    this.#queue.enqueue(['abort']);\n  }\n\n  status(s: DownstreamStatusMessage) {\n    this.#queue.enqueue(s);\n  }\n\n  catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n    this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n  }\n\n  #readyForMore: Resolver<void> | null = null;\n\n  readyForMore(): Promise<void> | undefined {\n    if (!this.#running) {\n      return undefined;\n    }\n    if (\n      this.#readyForMore === null &&\n      this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n    ) {\n      this.#lc.warn?.(\n        `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\\n` +\n          `\\n` +\n          `To inspect changeLog backlog in your change DB:\\n` +\n          `  SELECT\\n` +\n          `    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n          `    change->>'tag' AS operation,\\n` +\n          `    COUNT(*) AS count\\n` +\n          `  FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n          `  GROUP BY 1, 2\\n` +\n          `  ORDER BY 3 DESC\\n` +\n          `  LIMIT 20;`,\n      );\n      this.#readyForMore = resolver();\n    }\n    return this.#readyForMore?.promise;\n  }\n\n  #maybeReleaseBackPressure() {\n    if (\n      this.#readyForMore !== null &&\n      // Wait for at least 20% of the threshold to free up.\n      this.#approximateQueuedBytes < this.#backPressureThresholdBytes * 0.8\n    ) {\n      this.#lc.info?.(\n        `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`,\n      );\n      this.#readyForMore.resolve();\n      this.#readyForMore = null;\n    }\n  }\n\n  #stopped = promiseVoid;\n\n  /**\n   * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n   * Once {@link run()} completes, it can be called again.\n   */\n  async run() {\n    assert(!this.#running, `storer is already running`);\n\n    const {promise: stopped, resolve: signalStopped} = resolver();\n    this.#running = true;\n    this.#stopped = stopped;\n\n    this.#lc.info?.('starting storer');\n    let err: unknown;\n    try {\n      await this.#processQueue();\n    } catch (e) {\n      err = e; // used in finally\n      throw e;\n    } finally {\n      // Release any pending backpressure so the upstream can proceed\n      if (this.#readyForMore !== null) {\n        this.#readyForMore.resolve();\n        this.#readyForMore = null;\n      }\n      this.#cancelQueueEntries(\n        this.#queue.drain().filter(entry => entry !== undefined),\n        err,\n      );\n      this.#running = false;\n      signalStopped();\n      this.#lc.info?.('storer stopped');\n    }\n  }\n\n  #cancelQueueEntries(queue: QueueEntry[], e: unknown) {\n    if (queue.length === 0) {\n      return;\n    }\n    this.#lc.info?.(\n      `canceling ${queue.length} entries from the changeLog queue`,\n    );\n    const err = e instanceof Error ? e : new AbortError('server shutting down');\n    for (const entry of queue) {\n      if (entry === 'stop') {\n        continue;\n      }\n      const type = entry[0];\n      switch (type) {\n        case 'subscriber': {\n          // Disconnect subscribers waiting to be caught up so that they can\n          // reconnect and try again.\n          const {subscriber} = entry[1];\n          this.#lc.info?.(`disconnecting ${subscriber.id}`);\n          subscriber.fail(err);\n          break;\n        }\n      }\n    }\n  }\n\n  async #processQueue() {\n    let tx: PendingTransaction | null = null;\n    let msg: QueueEntry | false;\n\n    const catchupQueue: SubscriberAndMode[] = [];\n    try {\n      while ((msg = await this.#queue.dequeue()) !== 'stop') {\n        const [msgType] = msg;\n        switch (msgType) {\n          case 'ready': {\n            const signalReady = msg[1];\n            signalReady();\n            continue;\n          }\n          case 'subscriber': {\n            const subscriber = msg[1];\n            if (tx) {\n              catchupQueue.push(subscriber); // Wait for the current tx to complete.\n            } else {\n              await this.#startCatchup([subscriber]); // Catch up immediately.\n            }\n            continue;\n          }\n          case 'status':\n            this.#onConsumed(msg);\n            continue;\n          case 'abort': {\n            if (tx) {\n              tx.pool.abort();\n              await tx.pool.done();\n              tx = null;\n            }\n            continue;\n          }\n        }\n        // msgType === 'change'\n        const [_, watermark, json, change] = msg;\n        const tag = change?.tag;\n        this.#approximateQueuedBytes -= json.length;\n\n        if (tag === 'begin') {\n          assert(!tx, 'received BEGIN in the middle of a transaction');\n          const {promise, resolve, reject} = resolver<ReplicationState>();\n          void promise.catch(() => {}); // handle rejections before the await\n          tx = {\n            pool: new TransactionPool(\n              this.#lc.withContext('watermark', watermark),\n              {\n                mode: Mode.READ_COMMITTED,\n                statementResponseTimeout: this.#statementTimeoutMs,\n              },\n            ),\n            preCommitWatermark: watermark,\n            pos: 0,\n            startingReplicationState: promise,\n            ack: !change.skipAck,\n          };\n          tx.pool.run(this.#db);\n          // Acquire a lock on the replicationState row to detect and/or prevent\n          // a concurrent ownership change.\n          void tx.pool.process(tx => {\n            tx<ReplicationState[]> /*sql*/ `\n          SELECT * FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n              ([result]) => resolve(result),\n              reject,\n            );\n            return [];\n          });\n        } else {\n          assert(tx, () => `received change outside of transaction: ${json}`);\n          tx.pos++;\n        }\n\n        const entry = {\n          watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n          precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n          pos: tx.pos,\n          change: json,\n        };\n\n        const processed = tx.pool.process(sql => [\n          sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n          ...(change !== null && isSchemaChange(change)\n            ? this.#trackBackfillMetadata(sql, change)\n            : []),\n        ]);\n\n        if (tx.pos % 100 === 0) {\n          // Backpressure is exerted on commit when awaiting tx.pool.done().\n          // However, backpressure checks need to be regularly done for\n          // very large transactions in order to avoid memory blowup.\n          await processed;\n        }\n        this.#maybeReleaseBackPressure();\n\n        if (tag === 'commit') {\n          const {owner} = await tx.startingReplicationState;\n          if (owner !== this.#taskID) {\n            // Ownership change reflected in the replicationState read in 'begin'.\n            tx.pool.fail(\n              new AbortError(\n                `changeLog ownership has been assumed by ${owner}`,\n              ),\n            );\n          } else {\n            // Update the replication state.\n            const lastWatermark = watermark;\n            void tx.pool.process(tx => [\n              tx`\n            UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n            ]);\n            tx.pool.setDone();\n          }\n\n          await tx.pool.done();\n\n          // ACK the LSN to the upstream Postgres.\n          if (tx.ack) {\n            this.#onConsumed(['commit', change, {watermark}]);\n          }\n          tx = null;\n\n          // Before beginning the next transaction, open a READONLY snapshot to\n          // concurrently catchup any queued subscribers.\n          await this.#startCatchup(catchupQueue.splice(0));\n        } else if (tag === 'rollback') {\n          // Aborted transactions are not stored in the changeLog. Abort the current tx\n          // and process catchup of subscribers that were waiting for it to end.\n          tx.pool.abort();\n          await tx.pool.done();\n          tx = null;\n\n          await this.#startCatchup(catchupQueue.splice(0));\n        }\n      }\n    } catch (e) {\n      catchupQueue.forEach(({subscriber}) => subscriber.fail(e));\n      throw e;\n    }\n  }\n\n  async #startCatchup(subs: SubscriberAndMode[]) {\n    if (subs.length === 0) {\n      return;\n    }\n\n    const reader = new TransactionPool(\n      this.#lc.withContext('pool', 'catchup'),\n      {mode: Mode.READONLY},\n    );\n    reader.run(this.#db);\n\n    let lastWatermark: string | undefined;\n    try {\n      // Ensure that the transaction has started (and is thus holding a snapshot\n      // of the database) before continuing on to commit more changes. This is\n      // done by performing a single read on the db, which determines the\n      // snapshot for the REPEATABLE_READ transaction.\n      [{lastWatermark}] = await reader.processReadTask(\n        sql => sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')}\n      `,\n      );\n    } catch (e) {\n      subs.map(({subscriber}) => subscriber.fail(e));\n      throw e;\n    }\n\n    // Run the actual catchup queries in the background. Errors are handled in\n    // #catchup() by disconnecting the associated subscriber.\n    void Promise.all(\n      subs.map(sub => this.#catchup(sub, lastWatermark, reader)),\n    ).finally(() => reader.setDone());\n  }\n\n  async #catchup(\n    {subscriber: sub, mode}: SubscriberAndMode,\n    lastWatermark: string,\n    reader: TransactionPool,\n  ) {\n    try {\n      await reader.processReadTask(async tx => {\n        const start = Date.now();\n\n        // When starting from initial-sync, there won't be a change with a watermark\n        // equal to the replica version. This is the empty changeLog scenario.\n        let watermarkFound = sub.watermark === this.#replicaVersion;\n        let count = 0;\n        let lastBatchConsumed: Promise<unknown> | undefined;\n\n        for await (const entries of tx<ChangeEntry[]> /*sql*/ `\n          SELECT watermark, change FROM ${this.#cdc('changeLog')}\n           WHERE watermark >= ${sub.watermark}\n             AND watermark <= ${lastWatermark}\n           ORDER BY watermark, pos`.cursor(2000)) {\n          // Wait for the last batch of entries to be consumed by the\n          // subscriber before sending down the current batch. This pipelining\n          // allows one batch of changes to be received from the change-db\n          // while the previous batch of changes are sent to the subscriber,\n          // resulting in flow control that caps the number of changes\n          // referenced in memory to 2 * batch-size.\n          const start = performance.now();\n          await lastBatchConsumed;\n          const elapsed = performance.now() - start;\n          if (lastBatchConsumed) {\n            (elapsed > 100 ? this.#lc.info : this.#lc.debug)?.(\n              `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n            );\n          }\n\n          for (const entry of entries) {\n            if (entry.watermark === sub.watermark) {\n              // This should be the first entry.\n              // Catchup starts from *after* the watermark.\n              watermarkFound = true;\n            } else if (watermarkFound) {\n              lastBatchConsumed = sub.catchup(toDownstream(entry));\n              count++;\n            } else if (mode === 'backup') {\n              throw new AutoResetSignal(\n                `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n              );\n            } else {\n              this.#lc.warn?.(\n                `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n              );\n              sub.close(\n                ErrorType.WatermarkTooOld,\n                `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n              );\n              return;\n            }\n          }\n        }\n        if (watermarkFound) {\n          await lastBatchConsumed;\n          this.#lc.info?.(\n            `caught up ${sub.id} with ${count} changes (${\n              Date.now() - start\n            } ms)`,\n          );\n        } else {\n          this.#lc.warn?.(\n            `subscriber at watermark ${sub.watermark} is ahead of latest watermark`,\n          );\n        }\n        // Flushes the backlog of messages buffered during catchup and\n        // allows the subscription to forward subsequent messages immediately.\n        sub.setCaughtUp();\n      });\n    } catch (err) {\n      this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n      if (err instanceof AutoResetSignal) {\n        await markResetRequired(this.#db, this.#shard);\n        this.#onFatal(err);\n      }\n      sub.fail(err);\n    }\n  }\n\n  /**\n   * Returns the db statements necessary to track backfill and table metadata\n   * presented in the `change`, if any.\n   */\n  #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n    const stmts: PendingQuery<Row[]>[] = [];\n\n    switch (change.tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        stmts.push(this.#upsertTableMetadataStmt(sql, table, metadata));\n        break;\n      }\n\n      case 'create-table': {\n        const {spec, metadata, backfill} = change;\n        if (metadata) {\n          stmts.push(this.#upsertTableMetadataStmt(sql, spec, metadata));\n        }\n        if (backfill) {\n          Object.entries(backfill).forEach(([col, backfill]) => {\n            stmts.push(\n              this.#upsertColumnBackfillStmt(sql, spec, col, backfill),\n            );\n          });\n        }\n        break;\n      }\n\n      case 'rename-table': {\n        const {old} = change;\n        const row = {schema: change.new.schema, table: change.new.name};\n        stmts.push(\n          sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n          sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n        );\n        break;\n      }\n\n      case 'drop-table': {\n        const {\n          id: {schema, name},\n        } = change;\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('tableMetadata')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n        );\n        break;\n      }\n\n      case 'add-column': {\n        const {table, tableMetadata, column, backfill} = change;\n        if (tableMetadata) {\n          stmts.push(this.#upsertTableMetadataStmt(sql, table, tableMetadata));\n        }\n        if (backfill) {\n          stmts.push(\n            this.#upsertColumnBackfillStmt(sql, table, column.name, backfill),\n          );\n        }\n        break;\n      }\n\n      case 'update-column': {\n        const {\n          table: {schema, name: table},\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          stmts.push(\n            sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${newName}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${oldName}`,\n          );\n        }\n        break;\n      }\n\n      case 'drop-column': {\n        const {\n          table: {schema, name},\n          column,\n        } = change;\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${column}`,\n        );\n        break;\n      }\n\n      case 'backfill-completed': {\n        const {\n          relation: {schema, name: table, rowKey},\n          columns,\n        } = change;\n        const cols = [...rowKey.columns, ...columns];\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql(cols)}`,\n        );\n      }\n    }\n    return stmts;\n  }\n\n  #upsertTableMetadataStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    metadata: TableMetadata,\n  ) {\n    const row: TableMetadataRow = {schema, table, metadata};\n    return sql`\n        INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\") \n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  #upsertColumnBackfillStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    column: string,\n    backfill: BackfillID,\n  ) {\n    const row: BackfillingColumn = {schema, table, column, backfill};\n    return sql`\n        INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\", \"column\") \n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  /**\n   * Waits until all currently queued entries have been processed.\n   * This is only used in tests.\n   */\n  async allProcessed() {\n    if (this.#running) {\n      const {promise, resolve} = resolver();\n      this.#queue.enqueue(['ready', resolve]);\n      await promise;\n    }\n  }\n\n  stop() {\n    if (this.#running) {\n      this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n      this.#queue.enqueue('stop');\n    }\n    return this.#stopped;\n  }\n}\n\nfunction toDownstream(entry: ChangeEntry): WatermarkedChange {\n  const {watermark, change} = entry;\n  switch (change.tag) {\n    case 'begin':\n      return [watermark, ['begin', change, {commitWatermark: watermark}]];\n    case 'commit':\n      return [watermark, ['commit', change, {watermark}]];\n    case 'rollback':\n      return [watermark, ['rollback', change]];\n    default:\n      return [watermark, ['data', change]];\n  }\n}\n\nexport class PurgeLock {\n  readonly #lc: LogContext;\n  readonly #tx: TransactionPool;\n  readonly replicaVersion: string;\n  readonly minWatermark: string;\n\n  constructor(\n    lc: LogContext,\n    tx: TransactionPool,\n    replicaVersion: string,\n    watermark: string,\n  ) {\n    this.#lc = lc;\n    this.#tx = tx;\n    this.replicaVersion = replicaVersion;\n    this.minWatermark = watermark;\n  }\n\n  #released = false;\n\n  async release() {\n    if (this.#released) {\n      return;\n    }\n    this.#released = true;\n    this.#tx.setDone();\n    await this.#tx\n      .done()\n      .catch(e => this.#lc.warn?.(`error from purge-lock release`, e));\n    this.#lc.info?.(`released purge lock on ${this.minWatermark}`);\n  }\n}\n\nexport class PurgeLocker {\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #db: PostgresDB;\n\n  constructor(lc: LogContext, shard: ShardID, db: PostgresDB) {\n    this.#lc = lc.withContext('component', 'purge-locker');\n    this.#shard = shard;\n    this.#db = db;\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async acquire() {\n    const tx = new TransactionPool(this.#lc, {mode: Mode.READ_COMMITTED}).run(\n      this.#db,\n    );\n    const row = await tx.processReadTask(\n      sql => sql<{watermark: string}[]>`\n      SELECT watermark FROM ${this.#cdc('changeLog')}\n        ORDER BY watermark, pos LIMIT 1\n        FOR SHARE \n    `,\n    );\n    if (row.length === 0) {\n      this.#lc.info?.(`changeLog is empty. No rows to purge-lock.`);\n      tx.setDone();\n      await tx.done();\n      return null;\n    }\n    const [{watermark}] = row;\n    const [{replicaVersion}] = await tx.processReadTask(\n      sql => sql<{replicaVersion: string}[]>`\n        SELECT \"replicaVersion\" FROM ${this.#cdc('replicationConfig')}\n      `,\n    );\n    this.#lc.info?.(\n      `locked watermark ${watermark} from being purged from replica@${replicaVersion}`,\n    );\n    return new PurgeLock(this.#lc, tx, replicaVersion, watermark);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAwEA,IAAM,yBAAyB,eAAE,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqC7D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,OAAmB;CACzC;CACA;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA,EAAC,iCAAiC,sBAClC;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,aAAa;AACpD,QAAA,QAAc;AACd,QAAA,SAAe;AACf,QAAA,mBAAyB;AACzB,QAAA,oBAA0B;AAC1B,QAAA,KAAW;AACX,QAAA,iBAAuB;AACvB,QAAA,aAAmB;AACnB,QAAA,UAAgB;AAChB,QAAA,qBAA2B;EAE3B,MAAM,YAAY,mBAAmB;AACrC,QAAA,8BACG,UAAU,kBAAkB,UAAU,kBACvC;AAEF,QAAA,GAAS,OACP,gBAAgB,MAAA,6BAAmC,QAAQ,GAAG,QAAQ,EAAE,CAAC,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,EAAE,CAAC,iCAE/E,EAAC,WAAU,CACZ;;CAIH,KAAK,OAAe;AAClB,SAAO,MAAA,GAAS,GAAG,UAAU,MAAA,MAAY,CAAC,GAAG,QAAQ;;CAGvD,MAAM,gBAAgB,WAA8B;EAClD,MAAM,KAAK,MAAA;EACX,MAAM,QAAQ,MAAA;EACd,MAAM,eAAe,MAAA;EACrB,MAAM,gBAAgB,MAAA;EAEtB,MAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,cAAc,KAAK;AAC5B,QAAA,GAAS,OAAO,yBAAyB,sBAAsB;EAC/D,MAAM,QAAQ,YAAY,KAAK;AAC/B,QAAM,EAAE,UAAU,MAAA,IAAU,mBAAmB,CAAC,OAAO,GAAG;GAAC;GAAO,cAAc;GAAoB,CAAC;EACrG,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,QAAA,GAAS,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,MACzD;AAED,MAAI,UAKG,WAAU,SAAS;;CAI5B,MAAM,yCAGH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,UAAU,MAAM,MACxC,MAAA,KACA,QAAO,CACL,GAA8B;sCACA,MAAA,IAAU,mBAAmB,IAK3D,GAAG;;;;;;;;;iBASM,MAAA,IAAU,cAAc,CAAC;sBACpB,MAAA,IAAU,gBAAgB,CAAC;;;UAI1C,EACD,EAAC,MAAM,UAAc,CACtB;AAED,SAAO;GACL;GACA,kBAAkB,MAAQ,QAAQ,uBAAuB;GAC1D;;CAGH,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,MAAA,EACvB;qDACyC,MAAA,IAAU,YAAY;AACvE,SAAO;;CAGT,mBAAmB,WAAoC;AACrD,SAAO,MAAM,MAAA,IAAU,OAAM,QAAO;AAGlC,SAAM,GAA0B;kCACJ,MAAA,IAAU,YAAY,CAAC;;;;GASnD,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;wBAEhC,MAAA,IAAU,YAAY,CAAC,qBAAqB,UAAU;;;GAIxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;wBAC7B,MAAA,IAAU,mBAAmB,CAAC;AAChD,OAAI,UAAU,MAAA,OACZ,OAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,QACjF;AAEH,UAAO,OAAO,QAAQ;IACtB;;;;;CAMJ,MAAM,OAA0B;EAC9B,MAAM,CAAC,WAAW,CAAC,MAAM,WAAW;EAQpC,MAAM,OAAO,WAAW,UAAU,OAAO;AACzC,QAAA,0BAAgC,KAAK;AAErC,QAAA,MAAY,QAAQ;GAClB;GACA;GACA;GACA,aAAa,OAAO,GAAG,OAAO;GAC/B,CAAC;AAEF,SAAO,KAAK;;CAGd,QAAQ;AACN,QAAA,MAAY,QAAQ,CAAC,QAAQ,CAAC;;CAGhC,OAAO,GAA4B;AACjC,QAAA,MAAY,QAAQ,EAAE;;CAGxB,QAAQ,YAAwB,MAAsB;AACpD,QAAA,MAAY,QAAQ,CAAC,cAAc;GAAC;GAAY;GAAK,CAAC,CAAC;;CAGzD,gBAAuC;CAEvC,eAA0C;AACxC,MAAI,CAAC,MAAA,QACH;AAEF,MACE,MAAA,iBAAuB,QACvB,MAAA,yBAA+B,MAAA,4BAC/B;AACA,SAAA,GAAS,OACP,+BAA+B,MAAA,MAAY,MAAM,CAAC,qBAAqB,MAAA,yBAA+B,QAAQ,GAAG,QAAQ,EAAE,CAAC,4SAW7H;AACD,SAAA,eAAqB,UAAU;;AAEjC,SAAO,MAAA,cAAoB;;CAG7B,4BAA4B;AAC1B,MACE,MAAA,iBAAuB,QAEvB,MAAA,yBAA+B,MAAA,6BAAmC,IAClE;AACA,SAAA,GAAS,OACP,gCAAgC,MAAA,MAAY,MAAM,CAAC,qBAAqB,MAAA,yBAA+B,QAAQ,GAAG,QAAQ,EAAE,CAAC,MAC9H;AACD,SAAA,aAAmB,SAAS;AAC5B,SAAA,eAAqB;;;CAIzB,WAAW;;;;;CAMX,MAAM,MAAM;AACV,SAAO,CAAC,MAAA,SAAe,4BAA4B;EAEnD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,UAAU;AAC7D,QAAA,UAAgB;AAChB,QAAA,UAAgB;AAEhB,QAAA,GAAS,OAAO,kBAAkB;EAClC,IAAI;AACJ,MAAI;AACF,SAAM,MAAA,cAAoB;WACnB,GAAG;AACV,SAAM;AACN,SAAM;YACE;AAER,OAAI,MAAA,iBAAuB,MAAM;AAC/B,UAAA,aAAmB,SAAS;AAC5B,UAAA,eAAqB;;AAEvB,SAAA,mBACE,MAAA,MAAY,OAAO,CAAC,QAAO,UAAS,UAAU,KAAA,EAAU,EACxD,IACD;AACD,SAAA,UAAgB;AAChB,kBAAe;AACf,SAAA,GAAS,OAAO,iBAAiB;;;CAIrC,oBAAoB,OAAqB,GAAY;AACnD,MAAI,MAAM,WAAW,EACnB;AAEF,QAAA,GAAS,OACP,aAAa,MAAM,OAAO,mCAC3B;EACD,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,WAAW,uBAAuB;AAC3E,OAAK,MAAM,SAAS,OAAO;AACzB,OAAI,UAAU,OACZ;AAGF,WADa,MAAM,IACnB;IACE,KAAK,cAAc;KAGjB,MAAM,EAAC,eAAc,MAAM;AAC3B,WAAA,GAAS,OAAO,iBAAiB,WAAW,KAAK;AACjD,gBAAW,KAAK,IAAI;AACpB;;;;;CAMR,OAAA,eAAsB;EACpB,IAAI,KAAgC;EACpC,IAAI;EAEJ,MAAM,eAAoC,EAAE;AAC5C,MAAI;AACF,WAAQ,MAAM,MAAM,MAAA,MAAY,SAAS,MAAM,QAAQ;IACrD,MAAM,CAAC,WAAW;AAClB,YAAQ,SAAR;KACE,KAAK,SAAS;MACZ,MAAM,cAAc,IAAI;AACxB,mBAAa;AACb;;KAEF,KAAK,cAAc;MACjB,MAAM,aAAa,IAAI;AACvB,UAAI,GACF,cAAa,KAAK,WAAW;UAE7B,OAAM,MAAA,aAAmB,CAAC,WAAW,CAAC;AAExC;;KAEF,KAAK;AACH,YAAA,WAAiB,IAAI;AACrB;KACF,KAAK;AACH,UAAI,IAAI;AACN,UAAG,KAAK,OAAO;AACf,aAAM,GAAG,KAAK,MAAM;AACpB,YAAK;;AAEP;;IAIJ,MAAM,CAAC,GAAG,WAAW,MAAM,UAAU;IACrC,MAAM,MAAM,QAAQ;AACpB,UAAA,0BAAgC,KAAK;AAErC,QAAI,QAAQ,SAAS;AACnB,YAAO,CAAC,IAAI,gDAAgD;KAC5D,MAAM,EAAC,SAAS,SAAS,WAAU,UAA4B;AAC1D,aAAQ,YAAY,GAAG;AAC5B,UAAK;MACH,MAAM,IAAI,gBACR,MAAA,GAAS,YAAY,aAAa,UAAU,EAC5C;OACE,MAAM;OACN,0BAA0B,MAAA;OAC3B,CACF;MACD,oBAAoB;MACpB,KAAK;MACL,0BAA0B;MAC1B,KAAK,CAAC,OAAO;MACd;AACD,QAAG,KAAK,IAAI,MAAA,GAAS;AAGhB,QAAG,KAAK,SAAQ,OAAM;AACzB,QAA+B;0BACjB,MAAA,IAAU,mBAAmB,CAAC,aAAa,MACtD,CAAC,YAAY,QAAQ,OAAO,EAC7B,OACD;AACD,aAAO,EAAE;OACT;WACG;AACL,YAAO,UAAU,2CAA2C,OAAO;AACnE,QAAG;;IAGL,MAAM,QAAQ;KACZ,WAAW,QAAQ,WAAW,YAAY,GAAG;KAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;KACtD,KAAK,GAAG;KACR,QAAQ;KACT;IAED,MAAM,YAAY,GAAG,KAAK,SAAQ,QAAO,CACvC,GAAG,eAAe,MAAA,IAAU,YAAY,CAAC,GAAG,IAAI,MAAM,IACtD,GAAI,WAAW,QAAQ,eAAe,OAAO,GACzC,MAAA,sBAA4B,KAAK,OAAO,GACxC,EAAE,CACP,CAAC;AAEF,QAAI,GAAG,MAAM,QAAQ,EAInB,OAAM;AAER,UAAA,0BAAgC;AAEhC,QAAI,QAAQ,UAAU;KACpB,MAAM,EAAC,UAAS,MAAM,GAAG;AACzB,SAAI,UAAU,MAAA,OAEZ,IAAG,KAAK,KACN,IAAI,WACF,2CAA2C,QAC5C,CACF;UACI;MAEL,MAAM,gBAAgB;AACjB,SAAG,KAAK,SAAQ,OAAM,CACzB,EAAE;qBACK,MAAA,IAAU,mBAAmB,CAAC,OAAO,GAAG,EAAC,eAAc,CAAC,GAChE,CAAC;AACF,SAAG,KAAK,SAAS;;AAGnB,WAAM,GAAG,KAAK,MAAM;AAGpB,SAAI,GAAG,IACL,OAAA,WAAiB;MAAC;MAAU;MAAQ,EAAC,WAAU;MAAC,CAAC;AAEnD,UAAK;AAIL,WAAM,MAAA,aAAmB,aAAa,OAAO,EAAE,CAAC;eACvC,QAAQ,YAAY;AAG7B,QAAG,KAAK,OAAO;AACf,WAAM,GAAG,KAAK,MAAM;AACpB,UAAK;AAEL,WAAM,MAAA,aAAmB,aAAa,OAAO,EAAE,CAAC;;;WAG7C,GAAG;AACV,gBAAa,SAAS,EAAC,iBAAgB,WAAW,KAAK,EAAE,CAAC;AAC1D,SAAM;;;CAIV,OAAA,aAAoB,MAA2B;AAC7C,MAAI,KAAK,WAAW,EAClB;EAGF,MAAM,SAAS,IAAI,gBACjB,MAAA,GAAS,YAAY,QAAQ,UAAU,EACvC,EAAC,MAAM,UAAc,CACtB;AACD,SAAO,IAAI,MAAA,GAAS;EAEpB,IAAI;AACJ,MAAI;AAKF,IAAC,CAAC,kBAAkB,MAAM,OAAO,iBAC/B,QAAO,GAAuB;wBACd,MAAA,IAAU,mBAAmB,CAAC;QAE/C;WACM,GAAG;AACV,QAAK,KAAK,EAAC,iBAAgB,WAAW,KAAK,EAAE,CAAC;AAC9C,SAAM;;AAKH,UAAQ,IACX,KAAK,KAAI,QAAO,MAAA,QAAc,KAAK,eAAe,OAAO,CAAC,CAC3D,CAAC,cAAc,OAAO,SAAS,CAAC;;CAGnC,OAAA,QACE,EAAC,YAAY,KAAK,QAClB,eACA,QACA;AACA,MAAI;AACF,SAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,MAAM,QAAQ,KAAK,KAAK;IAIxB,IAAI,iBAAiB,IAAI,cAAc,MAAA;IACvC,IAAI,QAAQ;IACZ,IAAI;AAEJ,eAAW,MAAM,WAAW,EAA0B;0CACpB,MAAA,IAAU,YAAY,CAAC;gCACjC,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,IAAK,EAAE;KAOxC,MAAM,QAAQ,YAAY,KAAK;AAC/B,WAAM;KACN,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,SAAI,kBACF,EAAC,UAAU,MAAM,MAAA,GAAS,OAAO,MAAA,GAAS,SACxC,UAAU,QAAQ,QAAQ,EAAE,CAAC,UAAU,IAAI,GAAG,2CAC/C;AAGH,UAAK,MAAM,SAAS,QAClB,KAAI,MAAM,cAAc,IAAI,UAG1B,kBAAiB;cACR,gBAAgB;AACzB,0BAAoB,IAAI,QAAQ,aAAa,MAAM,CAAC;AACpD;gBACS,SAAS,SAClB,OAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,GACtF;UACI;AACL,YAAA,GAAS,OACP,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,GAC5F;AACD,UAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,GAChF;AACD;;;AAIN,QAAI,gBAAgB;AAClB,WAAM;AACN,WAAA,GAAS,OACP,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,KAAK,GAAG,MACd,MACF;UAED,OAAA,GAAS,OACP,2BAA2B,IAAI,UAAU,+BAC1C;AAIH,QAAI,aAAa;KACjB;WACK,KAAK;AACZ,SAAA,GAAS,QAAQ,sCAAsC,IAAI,MAAM,IAAI;AACrE,OAAI,eAAe,iBAAiB;AAClC,UAAM,kBAAkB,MAAA,IAAU,MAAA,MAAY;AAC9C,UAAA,QAAc,IAAI;;AAEpB,OAAI,KAAK,IAAI;;;;;;;CAQjB,uBAAuB,KAA0B,QAAsB;EACrE,MAAM,QAA+B,EAAE;AAEvC,UAAQ,OAAO,KAAf;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;AAC/B,UAAM,KAAK,MAAA,wBAA8B,KAAK,OAAO,SAAS,CAAC;AAC/D;;GAGF,KAAK,gBAAgB;IACnB,MAAM,EAAC,MAAM,UAAU,aAAY;AACnC,QAAI,SACF,OAAM,KAAK,MAAA,wBAA8B,KAAK,MAAM,SAAS,CAAC;AAEhE,QAAI,SACF,QAAO,QAAQ,SAAS,CAAC,SAAS,CAAC,KAAK,cAAc;AACpD,WAAM,KACJ,MAAA,yBAA+B,KAAK,MAAM,KAAK,SAAS,CACzD;MACD;AAEJ;;GAGF,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,OAAO,IAAI;KAAQ,OAAO,OAAO,IAAI;KAAK;AAC/D,UAAM,KACJ,GAAG,UAAU,MAAA,IAAU,gBAAgB,CAAC,OAAO,IAAI,IAAI,CAAC;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,MAAA,IAAU,cAAc,CAAC,OAAO,IAAI,IAAI,CAAC;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,OAC1D;AACD;;GAGF,KAAK,cAAc;IACjB,MAAM,EACJ,IAAI,EAAC,QAAQ,WACX;AACJ,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,gBAAgB,CAAC;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,OAClD;AACD;;GAGF,KAAK,cAAc;IACjB,MAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;AACjD,QAAI,cACF,OAAM,KAAK,MAAA,wBAA8B,KAAK,OAAO,cAAc,CAAC;AAEtE,QAAI,SACF,OAAM,KACJ,MAAA,yBAA+B,KAAK,OAAO,OAAO,MAAM,SAAS,CAClE;AAEH;;GAGF,KAAK,iBAAiB;IACpB,MAAM,EACJ,OAAO,EAAC,QAAQ,MAAM,SACtB,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;AACJ,QAAI,YAAY,QACd,OAAM,KACJ,GAAG,UAAU,MAAA,IAAU,cAAc,CAAC,kBAAkB,QAAQ;mCACzC,OAAO,iBAAiB,MAAM,kBAAkB,UACxE;AAEH;;GAGF,KAAK,eAAe;IAClB,MAAM,EACJ,OAAO,EAAC,QAAQ,QAChB,WACE;AACJ,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,SACzE;AACD;;GAGF,KAAK,sBAAsB;IACzB,MAAM,EACJ,UAAU,EAAC,QAAQ,MAAM,OAAO,UAChC,YACE;IACJ,MAAM,OAAO,CAAC,GAAG,OAAO,SAAS,GAAG,QAAQ;AAC5C,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,KAAK,GACpF;;;AAGL,SAAO;;CAGT,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;GAAS;AACvD,SAAO,GAAG;sBACQ,MAAA,IAAU,gBAAgB,CAAC,GAAG,IAAI,IAAI,CAAC;;0BAEnC,IAAI,IAAI,CAAC;;;CAIjC,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;GAAS;AAChE,SAAO,GAAG;sBACQ,MAAA,IAAU,cAAc,CAAC,GAAG,IAAI,IAAI,CAAC;;0BAEjC,IAAI,IAAI,CAAC;;;;;;;CAQjC,MAAM,eAAe;AACnB,MAAI,MAAA,SAAe;GACjB,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,SAAA,MAAY,QAAQ,CAAC,SAAS,QAAQ,CAAC;AACvC,SAAM;;;CAIV,OAAO;AACL,MAAI,MAAA,SAAe;AACjB,SAAA,GAAS,OAAO,YAAY,MAAA,MAAY,MAAM,CAAC,oBAAoB;AACnE,SAAA,MAAY,QAAQ,OAAO;;AAE7B,SAAO,MAAA;;;AAIX,SAAS,aAAa,OAAuC;CAC3D,MAAM,EAAC,WAAW,WAAU;AAC5B,SAAQ,OAAO,KAAf;EACE,KAAK,QACH,QAAO,CAAC,WAAW;GAAC;GAAS;GAAQ,EAAC,iBAAiB,WAAU;GAAC,CAAC;EACrE,KAAK,SACH,QAAO,CAAC,WAAW;GAAC;GAAU;GAAQ,EAAC,WAAU;GAAC,CAAC;EACrD,KAAK,WACH,QAAO,CAAC,WAAW,CAAC,YAAY,OAAO,CAAC;EAC1C,QACE,QAAO,CAAC,WAAW,CAAC,QAAQ,OAAO,CAAC;;;AAI1C,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,gBACA,WACA;AACA,QAAA,KAAW;AACX,QAAA,KAAW;AACX,OAAK,iBAAiB;AACtB,OAAK,eAAe;;CAGtB,YAAY;CAEZ,MAAM,UAAU;AACd,MAAI,MAAA,SACF;AAEF,QAAA,WAAiB;AACjB,QAAA,GAAS,SAAS;AAClB,QAAM,MAAA,GACH,MAAM,CACN,OAAM,MAAK,MAAA,GAAS,OAAO,iCAAiC,EAAE,CAAC;AAClE,QAAA,GAAS,OAAO,0BAA0B,KAAK,eAAe;;;AAIlE,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB,IAAgB;AAC1D,QAAA,KAAW,GAAG,YAAY,aAAa,eAAe;AACtD,QAAA,QAAc;AACd,QAAA,KAAW;;CAIb,KAAK,OAAe;AAClB,SAAO,MAAA,GAAS,GAAG,UAAU,MAAA,MAAY,CAAC,GAAG,QAAQ;;CAGvD,MAAM,UAAU;EACd,MAAM,KAAK,IAAI,gBAAgB,MAAA,IAAU,EAAC,MAAM,gBAAoB,CAAC,CAAC,IACpE,MAAA,GACD;EACD,MAAM,MAAM,MAAM,GAAG,iBACnB,QAAO,GAA0B;8BACT,MAAA,IAAU,YAAY,CAAC;;;MAIhD;AACD,MAAI,IAAI,WAAW,GAAG;AACpB,SAAA,GAAS,OAAO,6CAA6C;AAC7D,MAAG,SAAS;AACZ,SAAM,GAAG,MAAM;AACf,UAAO;;EAET,MAAM,CAAC,EAAC,eAAc;EACtB,MAAM,CAAC,EAAC,oBAAmB,MAAM,GAAG,iBAClC,QAAO,GAA+B;uCACL,MAAA,IAAU,oBAAoB,CAAC;QAEjE;AACD,QAAA,GAAS,OACP,oBAAoB,UAAU,kCAAkC,iBACjE;AACD,SAAO,IAAI,UAAU,MAAA,IAAU,IAAI,gBAAgB,UAAU"}
+{"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#db","#replicaVersion","#onConsumed","#onFatal","#queue","#backPressureThresholdBytes","#statementTimeoutMs","#cdc","#approximateQueuedBytes","#running","#readyForMore","#stopped","#processQueue","#cancelQueueEntries","#startCatchup","#trackBackfillMetadata","#maybeReleaseBackPressure","#catchup","#upsertTableMetadataStmt","#upsertColumnBackfillStmt","#tx","#released"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {getHeapStatistics} from 'node:v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {\n  backfillRequestSchema,\n  isDataChange,\n  isSchemaChange,\n  type BackfillID,\n  type BackfillRequest,\n  type Change,\n  type DataChange,\n  type Identifier,\n  type SchemaChange,\n  type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {type Commit} from '../change-source/protocol/current/downstream.ts';\nimport type {\n  DownstreamStatusMessage,\n  UpstreamStatusMessage,\n} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type ChangeEntry} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n  AutoResetSignal,\n  markResetRequired,\n  type BackfillingColumn,\n  type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n  subscriber: Subscriber;\n  mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n  | [\n      'change',\n      watermark: string,\n      json: string,\n      orig: Exclude<Change, DataChange> | null, // null for DataChanges\n    ]\n  | ['ready', callback: () => void]\n  | ['subscriber', SubscriberAndMode]\n  | DownstreamStatusMessage\n  | ['abort']\n  | 'stop';\n\ntype PendingTransaction = {\n  pool: TransactionPool;\n  preCommitWatermark: string;\n  pos: number;\n  startingReplicationState: Promise<ReplicationOwner>;\n  ack: boolean;\n};\n\ntype ReplicationOwner = {\n  owner: string | null;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\nexport type TuningOptions = {\n  backPressureLimitHeapProportion: number;\n  statementTimeoutMs: number;\n};\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n *   `replicaVersion`, which is the version at which initial sync\n *   was performed, and any attempts to catchup from a different\n *   point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n *   a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n *   subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n  readonly id = 'storer';\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #taskID: string;\n  readonly #discoveryAddress: string;\n  readonly #discoveryProtocol: string;\n  readonly #db: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #onConsumed: (c: Commit | UpstreamStatusMessage) => void;\n  readonly #onFatal: (err: Error) => void;\n  readonly #queue = new Queue<QueueEntry>();\n  readonly #backPressureThresholdBytes: number;\n  readonly #statementTimeoutMs: number;\n\n  #approximateQueuedBytes = 0;\n  #running = false;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    db: PostgresDB,\n    replicaVersion: string,\n    onConsumed: (c: Commit | UpstreamStatusMessage) => void,\n    onFatal: (err: Error) => void,\n    {backPressureLimitHeapProportion, statementTimeoutMs}: TuningOptions,\n  ) {\n    this.#lc = lc.withContext('component', 'change-log');\n    this.#shard = shard;\n    this.#taskID = taskID;\n    this.#discoveryAddress = discoveryAddress;\n    this.#discoveryProtocol = discoveryProtocol;\n    this.#db = db;\n    this.#replicaVersion = replicaVersion;\n    this.#onConsumed = onConsumed;\n    this.#onFatal = onFatal;\n    this.#statementTimeoutMs = statementTimeoutMs;\n\n    const heapStats = getHeapStatistics();\n    this.#backPressureThresholdBytes =\n      (heapStats.heap_size_limit - heapStats.used_heap_size) *\n      backPressureLimitHeapProportion;\n\n    this.#lc.info?.(\n      `Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of ` +\n        `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) ` +\n        `to absorb upstream spikes`,\n      {heapStats},\n    );\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async assumeOwnership(purgeLock?: PurgeLock | null) {\n    const db = this.#db;\n    const owner = this.#taskID;\n    const ownerAddress = this.#discoveryAddress;\n    const ownerProtocol = this.#discoveryProtocol;\n    // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n    const addressWithProtocol =\n      ownerProtocol === 'ws'\n        ? ownerAddress\n        : `${ownerProtocol}://${ownerAddress}`;\n    this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n    const start = performance.now();\n    await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n    const elapsed = (performance.now() - start).toFixed(2);\n    this.#lc.info?.(\n      `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n    );\n\n    if (purgeLock) {\n      // Once ownership has been assumed, any initial purge-lock preventing the\n      // purging of change-log records can be released, as a change-streamer\n      // that was attempting to purge records will correspondingly abort on the\n      // ownership check.\n      void purgeLock.release();\n    }\n  }\n\n  async getStartStreamInitializationParameters(): Promise<{\n    lastWatermark: string;\n    backfillRequests: BackfillRequest[];\n  }> {\n    const [[{lastWatermark}], result] = await runTx(\n      this.#db,\n      sql => [\n        sql<{lastWatermark: string}[]>`\n        SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n        // Formats a BackfillRequest using json_object_agg() to construct the\n        // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n        // to make it optional and possibly `null`.\n        sql`\n        SELECT \n            json_build_object(\n              'schema', b.\"schema\",\n              'name', b.\"table\",\n              'metadata', t.\"metadata\"\n            ) as \"table\",\n            json_object_agg(b.\"column\", b.\"backfill\") \n              as \"columns\"\n          FROM ${this.#cdc('backfilling')} as b\n          LEFT JOIN ${this.#cdc('tableMetadata')} as t\n          ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n          GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n        `,\n      ],\n      {mode: Mode.READONLY},\n    );\n\n    return {\n      lastWatermark,\n      backfillRequests: v.parse(result, backfillRequestsSchema),\n    };\n  }\n\n  async getMinWatermarkForCatchup(): Promise<string | null> {\n    const [{minWatermark}] = await this.#db<{minWatermark: string | null}[]>\n    /*sql*/ `\n      SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n    return minWatermark;\n  }\n\n  purgeRecordsBefore(watermark: string): Promise<number> {\n    return runTx(this.#db, async sql => {\n      // This NOWAIT pre-check is an optimization to abort the transaction\n      // (and release associated resources) early.\n      await sql<{watermark: string}[]>`\n          SELECT watermark FROM ${this.#cdc('changeLog')}\n            ORDER BY watermark, pos LIMIT 1\n            FOR UPDATE NOWAIT\n        `;\n      // If the row is purge-locked by an incoming replication-manager, it\n      // will assume ownership of the change-log before releasing the lock.\n      // This DELETE blocks until the lock is released, allowing the change\n      // in ownership to be reliably detected (and the transaction aborted)\n      // in the subsequent check.\n      const [{deleted}] = await sql<{deleted: bigint}[]>`\n        WITH purged AS (\n          DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n            RETURNING watermark, pos\n        ) SELECT COUNT(*) as deleted FROM purged;`;\n\n      const [{owner}] = await sql<ReplicationOwner[]>`\n        SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR SHARE`;\n      if (owner !== this.#taskID) {\n        throw new AbortError(\n          `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n        );\n      }\n      return Number(deleted);\n    });\n  }\n\n  /**\n   * @returns The size of the serialized entry, for memory / I/O estimations.\n   */\n  store(entry: WatermarkedChange) {\n    const [watermark, [_tag, change]] = entry;\n    // Eagerly stringify the JSON object so that the memory usage can be\n    // more accurately measured (i.e. without an extra object traversal and\n    // ad hoc memory counting heuristics).\n    //\n    // This essentially moves the stringify() computation out of the pg client,\n    // which is instead configured to pass `string` objects directly as JSON\n    // strings for JSON-valued columns (see TypeOptions.sendStringAsJson).\n    const json = BigIntJSON.stringify(change);\n    this.#approximateQueuedBytes += json.length;\n\n    this.#queue.enqueue([\n      'change',\n      watermark,\n      json,\n      isDataChange(change) ? null : change, // drop DataChanges to save memory\n    ]);\n\n    return json.length;\n  }\n\n  abort() {\n    this.#queue.enqueue(['abort']);\n  }\n\n  status(s: DownstreamStatusMessage) {\n    this.#queue.enqueue(s);\n  }\n\n  catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n    this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n  }\n\n  #readyForMore: Resolver<void> | null = null;\n\n  readyForMore(): Promise<void> | undefined {\n    if (!this.#running) {\n      return undefined;\n    }\n    if (\n      this.#readyForMore === null &&\n      this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n    ) {\n      this.#lc.warn?.(\n        `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\\n` +\n          `\\n` +\n          `To inspect changeLog backlog in your change DB:\\n` +\n          `  SELECT\\n` +\n          `    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\\n` +\n          `    change->>'tag' AS operation,\\n` +\n          `    COUNT(*) AS count\\n` +\n          `  FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n          `  GROUP BY 1, 2\\n` +\n          `  ORDER BY 3 DESC\\n` +\n          `  LIMIT 20;`,\n      );\n      this.#readyForMore = resolver();\n    }\n    return this.#readyForMore?.promise;\n  }\n\n  #maybeReleaseBackPressure() {\n    if (\n      this.#readyForMore !== null &&\n      // Wait for at least 20% of the threshold to free up.\n      this.#approximateQueuedBytes < this.#backPressureThresholdBytes * 0.8\n    ) {\n      this.#lc.info?.(\n        `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`,\n      );\n      this.#readyForMore.resolve();\n      this.#readyForMore = null;\n    }\n  }\n\n  #stopped = promiseVoid;\n\n  /**\n   * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n   * Once {@link run()} completes, it can be called again.\n   */\n  async run() {\n    assert(!this.#running, `storer is already running`);\n\n    const {promise: stopped, resolve: signalStopped} = resolver();\n    this.#running = true;\n    this.#stopped = stopped;\n\n    this.#lc.info?.('starting storer');\n    let err: unknown;\n    try {\n      await this.#processQueue();\n    } catch (e) {\n      err = e; // used in finally\n      throw e;\n    } finally {\n      // Release any pending backpressure so the upstream can proceed\n      if (this.#readyForMore !== null) {\n        this.#readyForMore.resolve();\n        this.#readyForMore = null;\n      }\n      this.#cancelQueueEntries(\n        this.#queue.drain().filter(entry => entry !== undefined),\n        err,\n      );\n      this.#running = false;\n      signalStopped();\n      this.#lc.info?.('storer stopped');\n    }\n  }\n\n  #cancelQueueEntries(queue: QueueEntry[], e: unknown) {\n    if (queue.length === 0) {\n      return;\n    }\n    this.#lc.info?.(\n      `canceling ${queue.length} entries from the changeLog queue`,\n    );\n    const err = e instanceof Error ? e : new AbortError('server shutting down');\n    for (const entry of queue) {\n      if (entry === 'stop') {\n        continue;\n      }\n      const type = entry[0];\n      switch (type) {\n        case 'subscriber': {\n          // Disconnect subscribers waiting to be caught up so that they can\n          // reconnect and try again.\n          const {subscriber} = entry[1];\n          this.#lc.info?.(`disconnecting ${subscriber.id}`);\n          subscriber.fail(err);\n          break;\n        }\n      }\n    }\n  }\n\n  async #processQueue() {\n    let tx: PendingTransaction | null = null;\n    let msg: QueueEntry | false;\n\n    const catchupQueue: SubscriberAndMode[] = [];\n    try {\n      while ((msg = await this.#queue.dequeue()) !== 'stop') {\n        const [msgType] = msg;\n        switch (msgType) {\n          case 'ready': {\n            const signalReady = msg[1];\n            signalReady();\n            continue;\n          }\n          case 'subscriber': {\n            const subscriber = msg[1];\n            if (tx) {\n              catchupQueue.push(subscriber); // Wait for the current tx to complete.\n            } else {\n              await this.#startCatchup([subscriber]); // Catch up immediately.\n            }\n            continue;\n          }\n          case 'status':\n            this.#onConsumed(msg);\n            continue;\n          case 'abort': {\n            if (tx) {\n              tx.pool.abort();\n              await tx.pool.done();\n              tx = null;\n            }\n            continue;\n          }\n        }\n        // msgType === 'change'\n        const [_, watermark, json, change] = msg;\n        const tag = change?.tag;\n        this.#approximateQueuedBytes -= json.length;\n\n        if (tag === 'begin') {\n          assert(!tx, 'received BEGIN in the middle of a transaction');\n          const {promise, resolve, reject} = resolver<ReplicationOwner>();\n          void promise.catch(() => {}); // handle rejections before the await\n          tx = {\n            pool: new TransactionPool(\n              this.#lc.withContext('watermark', watermark),\n              {\n                mode: Mode.READ_COMMITTED,\n                statementResponseTimeout: this.#statementTimeoutMs,\n              },\n            ),\n            preCommitWatermark: watermark,\n            pos: 0,\n            startingReplicationState: promise,\n            ack: !change.skipAck,\n          };\n          tx.pool.run(this.#db);\n          // Acquire a lock on the replicationState row to detect and/or prevent\n          // a concurrent ownership change.\n          void tx.pool.process(tx => {\n            tx<ReplicationOwner[]> /*sql*/ `\n          SELECT \"owner\" FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n              ([result]) => resolve(result),\n              reject,\n            );\n            return [];\n          });\n        } else {\n          assert(tx, () => `received change outside of transaction: ${json}`);\n          tx.pos++;\n        }\n\n        const entry = {\n          watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n          precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n          pos: tx.pos,\n          change: json,\n        };\n\n        const processed = tx.pool.process(sql => [\n          sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n          ...(change !== null && isSchemaChange(change)\n            ? this.#trackBackfillMetadata(sql, change)\n            : []),\n        ]);\n\n        if (tx.pos % 100 === 0) {\n          // Backpressure is exerted on commit when awaiting tx.pool.done().\n          // However, backpressure checks need to be regularly done for\n          // very large transactions in order to avoid memory blowup.\n          await processed;\n        }\n        this.#maybeReleaseBackPressure();\n\n        if (tag === 'commit') {\n          const {owner} = await tx.startingReplicationState;\n          if (owner !== this.#taskID) {\n            // Ownership change reflected in the replicationState read in 'begin'.\n            tx.pool.fail(\n              new AbortError(\n                `changeLog ownership has been assumed by ${owner}`,\n              ),\n            );\n          } else {\n            // Update the replication state.\n            const lastWatermark = watermark;\n            void tx.pool.process(tx => [\n              tx`\n            UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n            ]);\n            tx.pool.setDone();\n          }\n\n          await tx.pool.done();\n\n          // ACK the LSN to the upstream Postgres.\n          if (tx.ack) {\n            this.#onConsumed(['commit', change, {watermark}]);\n          }\n          tx = null;\n\n          // Before beginning the next transaction, open a READONLY snapshot to\n          // concurrently catchup any queued subscribers.\n          await this.#startCatchup(catchupQueue.splice(0));\n        } else if (tag === 'rollback') {\n          // Aborted transactions are not stored in the changeLog. Abort the current tx\n          // and process catchup of subscribers that were waiting for it to end.\n          tx.pool.abort();\n          await tx.pool.done();\n          tx = null;\n\n          await this.#startCatchup(catchupQueue.splice(0));\n        }\n      }\n    } catch (e) {\n      catchupQueue.forEach(({subscriber}) => subscriber.fail(e));\n      throw e;\n    }\n  }\n\n  async #startCatchup(subs: SubscriberAndMode[]) {\n    if (subs.length === 0) {\n      return;\n    }\n\n    const reader = new TransactionPool(\n      this.#lc.withContext('pool', 'catchup'),\n      {mode: Mode.READONLY},\n    );\n    reader.run(this.#db);\n\n    let lastWatermark: string | undefined;\n    try {\n      // Ensure that the transaction has started (and is thus holding a snapshot\n      // of the database) before continuing on to commit more changes. This is\n      // done by performing a single read on the db, which determines the\n      // snapshot for the REPEATABLE_READ transaction.\n      [{lastWatermark}] = await reader.processReadTask(\n        sql => sql<{lastWatermark: string}[]>`\n        SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}\n      `,\n      );\n    } catch (e) {\n      subs.map(({subscriber}) => subscriber.fail(e));\n      throw e;\n    }\n\n    // Run the actual catchup queries in the background. Errors are handled in\n    // #catchup() by disconnecting the associated subscriber.\n    void Promise.all(\n      subs.map(sub => this.#catchup(sub, lastWatermark, reader)),\n    ).finally(() => reader.setDone());\n  }\n\n  async #catchup(\n    {subscriber: sub, mode}: SubscriberAndMode,\n    lastWatermark: string,\n    reader: TransactionPool,\n  ) {\n    try {\n      await reader.processReadTask(async tx => {\n        const start = Date.now();\n\n        // When starting from initial-sync, there won't be a change with a watermark\n        // equal to the replica version. This is the empty changeLog scenario.\n        let watermarkFound = sub.watermark === this.#replicaVersion;\n        let count = 0;\n        let lastBatchConsumed: Promise<unknown> | undefined;\n\n        for await (const entries of tx<ChangeEntry[]> /*sql*/ `\n          SELECT watermark, change FROM ${this.#cdc('changeLog')}\n           WHERE watermark >= ${sub.watermark}\n             AND watermark <= ${lastWatermark}\n           ORDER BY watermark, pos`.cursor(2000)) {\n          // Wait for the last batch of entries to be consumed by the\n          // subscriber before sending down the current batch. This pipelining\n          // allows one batch of changes to be received from the change-db\n          // while the previous batch of changes are sent to the subscriber,\n          // resulting in flow control that caps the number of changes\n          // referenced in memory to 2 * batch-size.\n          const start = performance.now();\n          await lastBatchConsumed;\n          const elapsed = performance.now() - start;\n          if (lastBatchConsumed) {\n            (elapsed > 100 ? this.#lc.info : this.#lc.debug)?.(\n              `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n            );\n          }\n\n          for (const entry of entries) {\n            if (entry.watermark === sub.watermark) {\n              // This should be the first entry.\n              // Catchup starts from *after* the watermark.\n              watermarkFound = true;\n            } else if (watermarkFound) {\n              lastBatchConsumed = sub.catchup(toDownstream(entry));\n              count++;\n            } else if (mode === 'backup') {\n              throw new AutoResetSignal(\n                `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n              );\n            } else {\n              this.#lc.warn?.(\n                `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n              );\n              sub.close(\n                ErrorType.WatermarkTooOld,\n                `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n              );\n              return;\n            }\n          }\n        }\n        if (watermarkFound) {\n          await lastBatchConsumed;\n          this.#lc.info?.(\n            `caught up ${sub.id} with ${count} changes (${\n              Date.now() - start\n            } ms)`,\n          );\n        } else {\n          this.#lc.warn?.(\n            `subscriber at watermark ${sub.watermark} is ahead of latest watermark`,\n          );\n        }\n        // Flushes the backlog of messages buffered during catchup and\n        // allows the subscription to forward subsequent messages immediately.\n        sub.setCaughtUp();\n      });\n    } catch (err) {\n      this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n      if (err instanceof AutoResetSignal) {\n        await markResetRequired(this.#db, this.#shard);\n        this.#onFatal(err);\n      }\n      sub.fail(err);\n    }\n  }\n\n  /**\n   * Returns the db statements necessary to track backfill and table metadata\n   * presented in the `change`, if any.\n   */\n  #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n    const stmts: PendingQuery<Row[]>[] = [];\n\n    switch (change.tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        stmts.push(this.#upsertTableMetadataStmt(sql, table, metadata));\n        break;\n      }\n\n      case 'create-table': {\n        const {spec, metadata, backfill} = change;\n        if (metadata) {\n          stmts.push(this.#upsertTableMetadataStmt(sql, spec, metadata));\n        }\n        if (backfill) {\n          Object.entries(backfill).forEach(([col, backfill]) => {\n            stmts.push(\n              this.#upsertColumnBackfillStmt(sql, spec, col, backfill),\n            );\n          });\n        }\n        break;\n      }\n\n      case 'rename-table': {\n        const {old} = change;\n        const row = {schema: change.new.schema, table: change.new.name};\n        stmts.push(\n          sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n          sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n                WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n        );\n        break;\n      }\n\n      case 'drop-table': {\n        const {\n          id: {schema, name},\n        } = change;\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('tableMetadata')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n        );\n        break;\n      }\n\n      case 'add-column': {\n        const {table, tableMetadata, column, backfill} = change;\n        if (tableMetadata) {\n          stmts.push(this.#upsertTableMetadataStmt(sql, table, tableMetadata));\n        }\n        if (backfill) {\n          stmts.push(\n            this.#upsertColumnBackfillStmt(sql, table, column.name, backfill),\n          );\n        }\n        break;\n      }\n\n      case 'update-column': {\n        const {\n          table: {schema, name: table},\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          stmts.push(\n            sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${newName}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${oldName}`,\n          );\n        }\n        break;\n      }\n\n      case 'drop-column': {\n        const {\n          table: {schema, name},\n          column,\n        } = change;\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${column}`,\n        );\n        break;\n      }\n\n      case 'backfill-completed': {\n        const {\n          relation: {schema, name: table, rowKey},\n          columns,\n        } = change;\n        const cols = [...rowKey.columns, ...columns];\n        stmts.push(\n          sql`DELETE FROM ${this.#cdc('backfilling')}\n                WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql(cols)}`,\n        );\n      }\n    }\n    return stmts;\n  }\n\n  #upsertTableMetadataStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    metadata: TableMetadata,\n  ) {\n    const row: TableMetadataRow = {schema, table, metadata};\n    return sql`\n        INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\") \n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  #upsertColumnBackfillStmt(\n    sql: PostgresTransaction,\n    {schema, name: table}: Identifier,\n    column: string,\n    backfill: BackfillID,\n  ) {\n    const row: BackfillingColumn = {schema, table, column, backfill};\n    return sql`\n        INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n          ON CONFLICT (\"schema\", \"table\", \"column\") \n          DO UPDATE SET ${sql(row)};\n    `;\n  }\n\n  /**\n   * Waits until all currently queued entries have been processed.\n   * This is only used in tests.\n   */\n  async allProcessed() {\n    if (this.#running) {\n      const {promise, resolve} = resolver();\n      this.#queue.enqueue(['ready', resolve]);\n      await promise;\n    }\n  }\n\n  stop() {\n    if (this.#running) {\n      this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n      this.#queue.enqueue('stop');\n    }\n    return this.#stopped;\n  }\n}\n\nfunction toDownstream(entry: ChangeEntry): WatermarkedChange {\n  const {watermark, change} = entry;\n  switch (change.tag) {\n    case 'begin':\n      return [watermark, ['begin', change, {commitWatermark: watermark}]];\n    case 'commit':\n      return [watermark, ['commit', change, {watermark}]];\n    case 'rollback':\n      return [watermark, ['rollback', change]];\n    default:\n      return [watermark, ['data', change]];\n  }\n}\n\nexport class PurgeLock {\n  readonly #lc: LogContext;\n  readonly #tx: TransactionPool;\n  readonly replicaVersion: string;\n  readonly minWatermark: string;\n\n  constructor(\n    lc: LogContext,\n    tx: TransactionPool,\n    replicaVersion: string,\n    watermark: string,\n  ) {\n    this.#lc = lc;\n    this.#tx = tx;\n    this.replicaVersion = replicaVersion;\n    this.minWatermark = watermark;\n  }\n\n  #released = false;\n\n  async release() {\n    if (this.#released) {\n      return;\n    }\n    this.#released = true;\n    this.#tx.setDone();\n    await this.#tx\n      .done()\n      .catch(e => this.#lc.warn?.(`error from purge-lock release`, e));\n    this.#lc.info?.(`released purge lock on ${this.minWatermark}`);\n  }\n}\n\nexport class PurgeLocker {\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #db: PostgresDB;\n\n  constructor(lc: LogContext, shard: ShardID, db: PostgresDB) {\n    this.#lc = lc.withContext('component', 'purge-locker');\n    this.#shard = shard;\n    this.#db = db;\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async acquire() {\n    const tx = new TransactionPool(this.#lc, {mode: Mode.READ_COMMITTED}).run(\n      this.#db,\n    );\n    const row = await tx.processReadTask(\n      sql => sql<{watermark: string}[]>`\n      SELECT watermark FROM ${this.#cdc('changeLog')}\n        ORDER BY watermark, pos LIMIT 1\n        FOR SHARE \n    `,\n    );\n    if (row.length === 0) {\n      this.#lc.info?.(`changeLog is empty. No rows to purge-lock.`);\n      tx.setDone();\n      await tx.done();\n      return null;\n    }\n    const [{watermark}] = row;\n    const [{replicaVersion}] = await tx.processReadTask(\n      sql => sql<{replicaVersion: string}[]>`\n        SELECT \"replicaVersion\" FROM ${this.#cdc('replicationConfig')}\n      `,\n    );\n    this.#lc.info?.(\n      `locked watermark ${watermark} from being purged from replica@${replicaVersion}`,\n    );\n    return new PurgeLock(this.#lc, tx, replicaVersion, watermark);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA2EA,IAAM,yBAAyB,eAAE,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqC7D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,OAAmB;CACzC;CACA;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA,EAAC,iCAAiC,sBAClC;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,aAAa;AACpD,QAAA,QAAc;AACd,QAAA,SAAe;AACf,QAAA,mBAAyB;AACzB,QAAA,oBAA0B;AAC1B,QAAA,KAAW;AACX,QAAA,iBAAuB;AACvB,QAAA,aAAmB;AACnB,QAAA,UAAgB;AAChB,QAAA,qBAA2B;EAE3B,MAAM,YAAY,mBAAmB;AACrC,QAAA,8BACG,UAAU,kBAAkB,UAAU,kBACvC;AAEF,QAAA,GAAS,OACP,gBAAgB,MAAA,6BAAmC,QAAQ,GAAG,QAAQ,EAAE,CAAC,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,EAAE,CAAC,iCAE/E,EAAC,WAAU,CACZ;;CAIH,KAAK,OAAe;AAClB,SAAO,MAAA,GAAS,GAAG,UAAU,MAAA,MAAY,CAAC,GAAG,QAAQ;;CAGvD,MAAM,gBAAgB,WAA8B;EAClD,MAAM,KAAK,MAAA;EACX,MAAM,QAAQ,MAAA;EACd,MAAM,eAAe,MAAA;EACrB,MAAM,gBAAgB,MAAA;EAEtB,MAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,cAAc,KAAK;AAC5B,QAAA,GAAS,OAAO,yBAAyB,sBAAsB;EAC/D,MAAM,QAAQ,YAAY,KAAK;AAC/B,QAAM,EAAE,UAAU,MAAA,IAAU,mBAAmB,CAAC,OAAO,GAAG;GAAC;GAAO,cAAc;GAAoB,CAAC;EACrG,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,QAAA,GAAS,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,MACzD;AAED,MAAI,UAKG,WAAU,SAAS;;CAI5B,MAAM,yCAGH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,UAAU,MAAM,MACxC,MAAA,KACA,QAAO,CACL,GAA8B;sCACA,MAAA,IAAU,mBAAmB,IAK3D,GAAG;;;;;;;;;iBASM,MAAA,IAAU,cAAc,CAAC;sBACpB,MAAA,IAAU,gBAAgB,CAAC;;;UAI1C,EACD,EAAC,MAAM,UAAc,CACtB;AAED,SAAO;GACL;GACA,kBAAkB,MAAQ,QAAQ,uBAAuB;GAC1D;;CAGH,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,MAAA,EACvB;qDACyC,MAAA,IAAU,YAAY;AACvE,SAAO;;CAGT,mBAAmB,WAAoC;AACrD,SAAO,MAAM,MAAA,IAAU,OAAM,QAAO;AAGlC,SAAM,GAA0B;kCACJ,MAAA,IAAU,YAAY,CAAC;;;;GASnD,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;wBAEhC,MAAA,IAAU,YAAY,CAAC,qBAAqB,UAAU;;;GAIxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;8BACvB,MAAA,IAAU,mBAAmB,CAAC;AACtD,OAAI,UAAU,MAAA,OACZ,OAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,QACjF;AAEH,UAAO,OAAO,QAAQ;IACtB;;;;;CAMJ,MAAM,OAA0B;EAC9B,MAAM,CAAC,WAAW,CAAC,MAAM,WAAW;EAQpC,MAAM,OAAO,WAAW,UAAU,OAAO;AACzC,QAAA,0BAAgC,KAAK;AAErC,QAAA,MAAY,QAAQ;GAClB;GACA;GACA;GACA,aAAa,OAAO,GAAG,OAAO;GAC/B,CAAC;AAEF,SAAO,KAAK;;CAGd,QAAQ;AACN,QAAA,MAAY,QAAQ,CAAC,QAAQ,CAAC;;CAGhC,OAAO,GAA4B;AACjC,QAAA,MAAY,QAAQ,EAAE;;CAGxB,QAAQ,YAAwB,MAAsB;AACpD,QAAA,MAAY,QAAQ,CAAC,cAAc;GAAC;GAAY;GAAK,CAAC,CAAC;;CAGzD,gBAAuC;CAEvC,eAA0C;AACxC,MAAI,CAAC,MAAA,QACH;AAEF,MACE,MAAA,iBAAuB,QACvB,MAAA,yBAA+B,MAAA,4BAC/B;AACA,SAAA,GAAS,OACP,+BAA+B,MAAA,MAAY,MAAM,CAAC,qBAAqB,MAAA,yBAA+B,QAAQ,GAAG,QAAQ,EAAE,CAAC,4SAW7H;AACD,SAAA,eAAqB,UAAU;;AAEjC,SAAO,MAAA,cAAoB;;CAG7B,4BAA4B;AAC1B,MACE,MAAA,iBAAuB,QAEvB,MAAA,yBAA+B,MAAA,6BAAmC,IAClE;AACA,SAAA,GAAS,OACP,gCAAgC,MAAA,MAAY,MAAM,CAAC,qBAAqB,MAAA,yBAA+B,QAAQ,GAAG,QAAQ,EAAE,CAAC,MAC9H;AACD,SAAA,aAAmB,SAAS;AAC5B,SAAA,eAAqB;;;CAIzB,WAAW;;;;;CAMX,MAAM,MAAM;AACV,SAAO,CAAC,MAAA,SAAe,4BAA4B;EAEnD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,UAAU;AAC7D,QAAA,UAAgB;AAChB,QAAA,UAAgB;AAEhB,QAAA,GAAS,OAAO,kBAAkB;EAClC,IAAI;AACJ,MAAI;AACF,SAAM,MAAA,cAAoB;WACnB,GAAG;AACV,SAAM;AACN,SAAM;YACE;AAER,OAAI,MAAA,iBAAuB,MAAM;AAC/B,UAAA,aAAmB,SAAS;AAC5B,UAAA,eAAqB;;AAEvB,SAAA,mBACE,MAAA,MAAY,OAAO,CAAC,QAAO,UAAS,UAAU,KAAA,EAAU,EACxD,IACD;AACD,SAAA,UAAgB;AAChB,kBAAe;AACf,SAAA,GAAS,OAAO,iBAAiB;;;CAIrC,oBAAoB,OAAqB,GAAY;AACnD,MAAI,MAAM,WAAW,EACnB;AAEF,QAAA,GAAS,OACP,aAAa,MAAM,OAAO,mCAC3B;EACD,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,WAAW,uBAAuB;AAC3E,OAAK,MAAM,SAAS,OAAO;AACzB,OAAI,UAAU,OACZ;AAGF,WADa,MAAM,IACnB;IACE,KAAK,cAAc;KAGjB,MAAM,EAAC,eAAc,MAAM;AAC3B,WAAA,GAAS,OAAO,iBAAiB,WAAW,KAAK;AACjD,gBAAW,KAAK,IAAI;AACpB;;;;;CAMR,OAAA,eAAsB;EACpB,IAAI,KAAgC;EACpC,IAAI;EAEJ,MAAM,eAAoC,EAAE;AAC5C,MAAI;AACF,WAAQ,MAAM,MAAM,MAAA,MAAY,SAAS,MAAM,QAAQ;IACrD,MAAM,CAAC,WAAW;AAClB,YAAQ,SAAR;KACE,KAAK,SAAS;MACZ,MAAM,cAAc,IAAI;AACxB,mBAAa;AACb;;KAEF,KAAK,cAAc;MACjB,MAAM,aAAa,IAAI;AACvB,UAAI,GACF,cAAa,KAAK,WAAW;UAE7B,OAAM,MAAA,aAAmB,CAAC,WAAW,CAAC;AAExC;;KAEF,KAAK;AACH,YAAA,WAAiB,IAAI;AACrB;KACF,KAAK;AACH,UAAI,IAAI;AACN,UAAG,KAAK,OAAO;AACf,aAAM,GAAG,KAAK,MAAM;AACpB,YAAK;;AAEP;;IAIJ,MAAM,CAAC,GAAG,WAAW,MAAM,UAAU;IACrC,MAAM,MAAM,QAAQ;AACpB,UAAA,0BAAgC,KAAK;AAErC,QAAI,QAAQ,SAAS;AACnB,YAAO,CAAC,IAAI,gDAAgD;KAC5D,MAAM,EAAC,SAAS,SAAS,WAAU,UAA4B;AAC1D,aAAQ,YAAY,GAAG;AAC5B,UAAK;MACH,MAAM,IAAI,gBACR,MAAA,GAAS,YAAY,aAAa,UAAU,EAC5C;OACE,MAAM;OACN,0BAA0B,MAAA;OAC3B,CACF;MACD,oBAAoB;MACpB,KAAK;MACL,0BAA0B;MAC1B,KAAK,CAAC,OAAO;MACd;AACD,QAAG,KAAK,IAAI,MAAA,GAAS;AAGhB,QAAG,KAAK,SAAQ,OAAM;AACzB,QAA+B;gCACX,MAAA,IAAU,mBAAmB,CAAC,aAAa,MAC5D,CAAC,YAAY,QAAQ,OAAO,EAC7B,OACD;AACD,aAAO,EAAE;OACT;WACG;AACL,YAAO,UAAU,2CAA2C,OAAO;AACnE,QAAG;;IAGL,MAAM,QAAQ;KACZ,WAAW,QAAQ,WAAW,YAAY,GAAG;KAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;KACtD,KAAK,GAAG;KACR,QAAQ;KACT;IAED,MAAM,YAAY,GAAG,KAAK,SAAQ,QAAO,CACvC,GAAG,eAAe,MAAA,IAAU,YAAY,CAAC,GAAG,IAAI,MAAM,IACtD,GAAI,WAAW,QAAQ,eAAe,OAAO,GACzC,MAAA,sBAA4B,KAAK,OAAO,GACxC,EAAE,CACP,CAAC;AAEF,QAAI,GAAG,MAAM,QAAQ,EAInB,OAAM;AAER,UAAA,0BAAgC;AAEhC,QAAI,QAAQ,UAAU;KACpB,MAAM,EAAC,UAAS,MAAM,GAAG;AACzB,SAAI,UAAU,MAAA,OAEZ,IAAG,KAAK,KACN,IAAI,WACF,2CAA2C,QAC5C,CACF;UACI;MAEL,MAAM,gBAAgB;AACjB,SAAG,KAAK,SAAQ,OAAM,CACzB,EAAE;qBACK,MAAA,IAAU,mBAAmB,CAAC,OAAO,GAAG,EAAC,eAAc,CAAC,GAChE,CAAC;AACF,SAAG,KAAK,SAAS;;AAGnB,WAAM,GAAG,KAAK,MAAM;AAGpB,SAAI,GAAG,IACL,OAAA,WAAiB;MAAC;MAAU;MAAQ,EAAC,WAAU;MAAC,CAAC;AAEnD,UAAK;AAIL,WAAM,MAAA,aAAmB,aAAa,OAAO,EAAE,CAAC;eACvC,QAAQ,YAAY;AAG7B,QAAG,KAAK,OAAO;AACf,WAAM,GAAG,KAAK,MAAM;AACpB,UAAK;AAEL,WAAM,MAAA,aAAmB,aAAa,OAAO,EAAE,CAAC;;;WAG7C,GAAG;AACV,gBAAa,SAAS,EAAC,iBAAgB,WAAW,KAAK,EAAE,CAAC;AAC1D,SAAM;;;CAIV,OAAA,aAAoB,MAA2B;AAC7C,MAAI,KAAK,WAAW,EAClB;EAGF,MAAM,SAAS,IAAI,gBACjB,MAAA,GAAS,YAAY,QAAQ,UAAU,EACvC,EAAC,MAAM,UAAc,CACtB;AACD,SAAO,IAAI,MAAA,GAAS;EAEpB,IAAI;AACJ,MAAI;AAKF,IAAC,CAAC,kBAAkB,MAAM,OAAO,iBAC/B,QAAO,GAA8B;sCACP,MAAA,IAAU,mBAAmB,CAAC;QAE7D;WACM,GAAG;AACV,QAAK,KAAK,EAAC,iBAAgB,WAAW,KAAK,EAAE,CAAC;AAC9C,SAAM;;AAKH,UAAQ,IACX,KAAK,KAAI,QAAO,MAAA,QAAc,KAAK,eAAe,OAAO,CAAC,CAC3D,CAAC,cAAc,OAAO,SAAS,CAAC;;CAGnC,OAAA,QACE,EAAC,YAAY,KAAK,QAClB,eACA,QACA;AACA,MAAI;AACF,SAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,MAAM,QAAQ,KAAK,KAAK;IAIxB,IAAI,iBAAiB,IAAI,cAAc,MAAA;IACvC,IAAI,QAAQ;IACZ,IAAI;AAEJ,eAAW,MAAM,WAAW,EAA0B;0CACpB,MAAA,IAAU,YAAY,CAAC;gCACjC,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,IAAK,EAAE;KAOxC,MAAM,QAAQ,YAAY,KAAK;AAC/B,WAAM;KACN,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,SAAI,kBACF,EAAC,UAAU,MAAM,MAAA,GAAS,OAAO,MAAA,GAAS,SACxC,UAAU,QAAQ,QAAQ,EAAE,CAAC,UAAU,IAAI,GAAG,2CAC/C;AAGH,UAAK,MAAM,SAAS,QAClB,KAAI,MAAM,cAAc,IAAI,UAG1B,kBAAiB;cACR,gBAAgB;AACzB,0BAAoB,IAAI,QAAQ,aAAa,MAAM,CAAC;AACpD;gBACS,SAAS,SAClB,OAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,GACtF;UACI;AACL,YAAA,GAAS,OACP,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,GAC5F;AACD,UAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,GAChF;AACD;;;AAIN,QAAI,gBAAgB;AAClB,WAAM;AACN,WAAA,GAAS,OACP,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,KAAK,GAAG,MACd,MACF;UAED,OAAA,GAAS,OACP,2BAA2B,IAAI,UAAU,+BAC1C;AAIH,QAAI,aAAa;KACjB;WACK,KAAK;AACZ,SAAA,GAAS,QAAQ,sCAAsC,IAAI,MAAM,IAAI;AACrE,OAAI,eAAe,iBAAiB;AAClC,UAAM,kBAAkB,MAAA,IAAU,MAAA,MAAY;AAC9C,UAAA,QAAc,IAAI;;AAEpB,OAAI,KAAK,IAAI;;;;;;;CAQjB,uBAAuB,KAA0B,QAAsB;EACrE,MAAM,QAA+B,EAAE;AAEvC,UAAQ,OAAO,KAAf;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;AAC/B,UAAM,KAAK,MAAA,wBAA8B,KAAK,OAAO,SAAS,CAAC;AAC/D;;GAGF,KAAK,gBAAgB;IACnB,MAAM,EAAC,MAAM,UAAU,aAAY;AACnC,QAAI,SACF,OAAM,KAAK,MAAA,wBAA8B,KAAK,MAAM,SAAS,CAAC;AAEhE,QAAI,SACF,QAAO,QAAQ,SAAS,CAAC,SAAS,CAAC,KAAK,cAAc;AACpD,WAAM,KACJ,MAAA,yBAA+B,KAAK,MAAM,KAAK,SAAS,CACzD;MACD;AAEJ;;GAGF,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,OAAO,IAAI;KAAQ,OAAO,OAAO,IAAI;KAAK;AAC/D,UAAM,KACJ,GAAG,UAAU,MAAA,IAAU,gBAAgB,CAAC,OAAO,IAAI,IAAI,CAAC;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,MAAA,IAAU,cAAc,CAAC,OAAO,IAAI,IAAI,CAAC;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,OAC1D;AACD;;GAGF,KAAK,cAAc;IACjB,MAAM,EACJ,IAAI,EAAC,QAAQ,WACX;AACJ,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,gBAAgB,CAAC;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,OAClD;AACD;;GAGF,KAAK,cAAc;IACjB,MAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;AACjD,QAAI,cACF,OAAM,KAAK,MAAA,wBAA8B,KAAK,OAAO,cAAc,CAAC;AAEtE,QAAI,SACF,OAAM,KACJ,MAAA,yBAA+B,KAAK,OAAO,OAAO,MAAM,SAAS,CAClE;AAEH;;GAGF,KAAK,iBAAiB;IACpB,MAAM,EACJ,OAAO,EAAC,QAAQ,MAAM,SACtB,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;AACJ,QAAI,YAAY,QACd,OAAM,KACJ,GAAG,UAAU,MAAA,IAAU,cAAc,CAAC,kBAAkB,QAAQ;mCACzC,OAAO,iBAAiB,MAAM,kBAAkB,UACxE;AAEH;;GAGF,KAAK,eAAe;IAClB,MAAM,EACJ,OAAO,EAAC,QAAQ,QAChB,WACE;AACJ,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,SACzE;AACD;;GAGF,KAAK,sBAAsB;IACzB,MAAM,EACJ,UAAU,EAAC,QAAQ,MAAM,OAAO,UAChC,YACE;IACJ,MAAM,OAAO,CAAC,GAAG,OAAO,SAAS,GAAG,QAAQ;AAC5C,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,KAAK,GACpF;;;AAGL,SAAO;;CAGT,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;GAAS;AACvD,SAAO,GAAG;sBACQ,MAAA,IAAU,gBAAgB,CAAC,GAAG,IAAI,IAAI,CAAC;;0BAEnC,IAAI,IAAI,CAAC;;;CAIjC,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;GAAS;AAChE,SAAO,GAAG;sBACQ,MAAA,IAAU,cAAc,CAAC,GAAG,IAAI,IAAI,CAAC;;0BAEjC,IAAI,IAAI,CAAC;;;;;;;CAQjC,MAAM,eAAe;AACnB,MAAI,MAAA,SAAe;GACjB,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,SAAA,MAAY,QAAQ,CAAC,SAAS,QAAQ,CAAC;AACvC,SAAM;;;CAIV,OAAO;AACL,MAAI,MAAA,SAAe;AACjB,SAAA,GAAS,OAAO,YAAY,MAAA,MAAY,MAAM,CAAC,oBAAoB;AACnE,SAAA,MAAY,QAAQ,OAAO;;AAE7B,SAAO,MAAA;;;AAIX,SAAS,aAAa,OAAuC;CAC3D,MAAM,EAAC,WAAW,WAAU;AAC5B,SAAQ,OAAO,KAAf;EACE,KAAK,QACH,QAAO,CAAC,WAAW;GAAC;GAAS;GAAQ,EAAC,iBAAiB,WAAU;GAAC,CAAC;EACrE,KAAK,SACH,QAAO,CAAC,WAAW;GAAC;GAAU;GAAQ,EAAC,WAAU;GAAC,CAAC;EACrD,KAAK,WACH,QAAO,CAAC,WAAW,CAAC,YAAY,OAAO,CAAC;EAC1C,QACE,QAAO,CAAC,WAAW,CAAC,QAAQ,OAAO,CAAC;;;AAI1C,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,gBACA,WACA;AACA,QAAA,KAAW;AACX,QAAA,KAAW;AACX,OAAK,iBAAiB;AACtB,OAAK,eAAe;;CAGtB,YAAY;CAEZ,MAAM,UAAU;AACd,MAAI,MAAA,SACF;AAEF,QAAA,WAAiB;AACjB,QAAA,GAAS,SAAS;AAClB,QAAM,MAAA,GACH,MAAM,CACN,OAAM,MAAK,MAAA,GAAS,OAAO,iCAAiC,EAAE,CAAC;AAClE,QAAA,GAAS,OAAO,0BAA0B,KAAK,eAAe;;;AAIlE,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB,IAAgB;AAC1D,QAAA,KAAW,GAAG,YAAY,aAAa,eAAe;AACtD,QAAA,QAAc;AACd,QAAA,KAAW;;CAIb,KAAK,OAAe;AAClB,SAAO,MAAA,GAAS,GAAG,UAAU,MAAA,MAAY,CAAC,GAAG,QAAQ;;CAGvD,MAAM,UAAU;EACd,MAAM,KAAK,IAAI,gBAAgB,MAAA,IAAU,EAAC,MAAM,gBAAoB,CAAC,CAAC,IACpE,MAAA,GACD;EACD,MAAM,MAAM,MAAM,GAAG,iBACnB,QAAO,GAA0B;8BACT,MAAA,IAAU,YAAY,CAAC;;;MAIhD;AACD,MAAI,IAAI,WAAW,GAAG;AACpB,SAAA,GAAS,OAAO,6CAA6C;AAC7D,MAAG,SAAS;AACZ,SAAM,GAAG,MAAM;AACf,UAAO;;EAET,MAAM,CAAC,EAAC,eAAc;EACtB,MAAM,CAAC,EAAC,oBAAmB,MAAM,GAAG,iBAClC,QAAO,GAA+B;uCACL,MAAA,IAAU,oBAAoB,CAAC;QAEjE;AACD,QAAA,GAAS,OACP,oBAAoB,UAAU,kCAAkC,iBACjE;AACD,SAAO,IAAI,UAAU,MAAA,IAAU,IAAI,gBAAgB,UAAU"}

package/out/zero-cache/src/services/replicator/schema/change-log.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-log.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/change-log.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AACzD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,iCAAiC,CAAC;AACzE,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAGvD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,eAAO,MAAM,MAAM,MAAM,CAAC;AAC1B,eAAO,MAAM,MAAM,MAAM,CAAC;AAC1B,eAAO,MAAM,WAAW,MAAM,CAAC;AAC/B,eAAO,MAAM,QAAQ,MAAM,CAAC;AAG5B,eAAO,MAAM,uBAAuB,sbAmCjC,CAAC;AAEJ;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB;;;;;EAc5B,CAAC;AAEN,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,QAAA,MAAM,uBAAuB;;;;;;aAQ3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,qBAAa,SAAS;;gBAMR,EAAE,EAAE,QAAQ;IAsCxB;;;;;;;;;;OAUG;IACH,QAAQ,CACN,OAAO,EAAE,WAAW,EACpB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,UAAU,EACf,UAAU,EAAE,MAAM,EAAE,GAAG,SAAS,GAC/B,MAAM;IAIT,WAAW,CACT,OAAO,EAAE,WAAW,EACpB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,UAAU,GACd,MAAM;IAOT,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU;;;;;;;IAoC7C,aAAa,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM;IAIjD,UAAU,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM;CAG/C"}
+{"version":3,"file":"change-log.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/change-log.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AACzD,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,iCAAiC,CAAC;AACzE,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAGvD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,eAAO,MAAM,MAAM,MAAM,CAAC;AAC1B,eAAO,MAAM,MAAM,MAAM,CAAC;AAC1B,eAAO,MAAM,WAAW,MAAM,CAAC;AAC/B,eAAO,MAAM,QAAQ,MAAM,CAAC;AAG5B,eAAO,MAAM,uBAAuB,sbAmCjC,CAAC;AAEJ;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB;;;;;EAc5B,CAAC;AAEN,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,QAAA,MAAM,uBAAuB;;;;;;aAQ3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,qBAAa,SAAS;;gBAMR,EAAE,EAAE,QAAQ;IAuCxB;;;;;;;;;;OAUG;IACH,QAAQ,CACN,OAAO,EAAE,WAAW,EACpB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,UAAU,EACf,UAAU,EAAE,MAAM,EAAE,GAAG,SAAS,GAC/B,MAAM;IAIT,WAAW,CACT,OAAO,EAAE,WAAW,EACpB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,UAAU,GACd,MAAM;IAOT,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU;;;;;;;IAoC7C,aAAa,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM;IAIjD,UAAU,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM;CAG/C"}

package/out/zero-cache/src/services/replicator/schema/change-log.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-log.js","names":["#logRowOpStmt","#logRowOpWithBackfillStmt","#logTableWideOpStmt","#getRowOpStmt","#logRowOp"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/change-log.ts"],"sourcesContent":["import {\n  jsonObjectSchema,\n  parse,\n  stringify,\n} from '../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport type {LexiVersion} from '../../../types/lexi-version.ts';\nimport type {LiteRowKey} from '../../../types/lite.ts';\nimport {normalizedKeyOrder} from '../../../types/row-key.ts';\n\n/**\n * The Change Log tracks the last operation (set or delete) for each row in the\n * data base, ordered by state version; in other words, a cross-table\n * index of row changes ordered by version. This facilitates a minimal \"diff\"\n * of row changes needed to advance a pipeline from one state version to another.\n *\n * The Change Log stores identifiers only, i.e. it does not store contents.\n * A database snapshot at the previous version can be used to query a row's\n * old contents, if any, and the current snapshot can be used to query a row's\n * new contents. (In the common case, the new contents will have just been applied\n * and thus has a high likelihood of being in the SQLite cache.)\n *\n * There are two table-wide operations:\n * - `t` corresponds to the postgres `TRUNCATE` operation\n * - `r` represents any schema (i.e. column) change\n *\n * For both operations, the corresponding row changes are not explicitly included\n * in the change log. The consumer has the option of simulating them be reading\n * from pre- and post- snapshots, or resetting their state entirely with the current\n * snapshot.\n *\n * To achieve the desired ordering semantics when processing tables that have been\n * truncated, reset, and modified, the \"rowKey\" is set to `null` for resets and\n * the empty string `\"\"` for truncates. This means that resets will be encountered\n * before truncates, which will be processed before any subsequent row changes.\n *\n * This ordering is chosen because resets are currently the more \"destructive\" op\n * and result in aborting the processing (and starting from scratch); doing this\n * earlier reduces wasted work.\n */\n\nexport const SET_OP = 's';\nexport const DEL_OP = 'd';\nexport const TRUNCATE_OP = 't';\nexport const RESET_OP = 'r';\n\n// Exported for testing (and migrations)\nexport const CREATE_CHANGELOG_SCHEMA =\n  // stateVersion : a.k.a. row version\n  // pos          : order in which to process the change (within the version)\n  // table        : The table associated with the change\n  // rowKey       : JSON row key for a row change. For table-wide changes RESET\n  //                and TRUNCATE, there is no associated row; instead, `pos` is\n  //                set to -1 and the rowKey is set to the stateVersion,\n  //                guaranteeing when attempting to process the transaction,\n  //                the pipeline is reset (and the change log traversal\n  //                aborted).\n  // op           : 's' for set (insert/update)\n  //              : 'd' for delete\n  //              : 'r' for table reset (schema change)\n  //              : 't' for table truncation (which also resets the pipeline)\n  // backfillingColumnVersions\n  //              : A JSON mapping from column name to stateVersion tracked\n  //                for replicated writes of columns that are being backfilled.\n  //                This is used to prevent backfill data, which is at a\n  //                fixed snapshot/version outside of the replication stream,\n  //                from overwriting newer column values.\n  //\n  // Naming note: To maintain compatibility between a new replication-manager\n  // and old view-syncers, the previous _zero.changeLog table is preserved\n  // and its replacement given a new name \"changeLog2\".\n  `\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    \"backfillingColumnVersions\" TEXT DEFAULT '{}',\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n  `;\n\n/**\n * Contains the changeLog fields relevant for computing the diff between\n * two snapshots of a replica. The `pos` and `backfillingColumnVersions`\n * fields are excluded, though the query should be ordered by\n * `<stateVersion, pos>`.\n */\nexport const changeLogEntrySchema = v\n  .object({\n    stateVersion: v.string(),\n    table: v.string(),\n    rowKey: v.string(),\n    op: v.literalUnion(SET_OP, DEL_OP, TRUNCATE_OP, RESET_OP),\n  })\n  .map(val => ({\n    ...val,\n    // Note: sets the rowKey to `null` for table-wide ops / resets\n    rowKey:\n      val.op === 't' || val.op === 'r'\n        ? null\n        : v.parse(parse(val.rowKey), jsonObjectSchema),\n  }));\n\nexport type ChangeLogEntry = v.Infer<typeof changeLogEntrySchema>;\n\nconst rawChangeLogEntrySchema = v.object({\n  stateVersion: v.string(),\n  table: v.string(),\n  rowKey: v.string(),\n  op: v.literalUnion(SET_OP, DEL_OP, TRUNCATE_OP, RESET_OP),\n  backfillingColumnVersions: v\n    .string()\n    .map(val => v.record(v.string()).parse(JSON.parse(val))),\n});\n\nexport type RawChangeLogEntry = v.Infer<typeof rawChangeLogEntrySchema>;\n\nexport class ChangeLog {\n  readonly #logRowOpStmt: Statement;\n  readonly #logRowOpWithBackfillStmt: Statement;\n  readonly #logTableWideOpStmt;\n  readonly #getRowOpStmt: Statement;\n\n  constructor(db: Database) {\n    this.#logRowOpStmt = db.prepare(/*sql*/ `\n      INSERT OR REPLACE INTO \"_zero.changeLog2\" \n        (stateVersion, pos, \"table\", rowKey, op)\n        VALUES (@version, @pos, @table, JSON(@rowKey), @op)\n    `);\n\n    this.#logRowOpWithBackfillStmt = db.prepare(/*sql*/ `\n      INSERT INTO \"_zero.changeLog2\" \n        (stateVersion, pos, \"table\", rowKey, op, backfillingColumnVersions)\n        VALUES (@version, @pos, @table, JSON(@rowKey), @op, \n                JSON(@backfillingColumnVersions))\n        ON CONFLICT (\"table\", rowKey) DO UPDATE \n                   SET stateVersion = excluded.stateVersion,\n                                pos = excluded.pos,\n                                 op = excluded.op,\n          backfillingColumnVersions = json_patch(\n          backfillingColumnVersions, excluded.backfillingColumnVersions)\n    `);\n\n    // Because table-wide ops result in aborting an incremental update\n    // and rehydrating all queries at \"head\", they are assigned pos = -1\n    // as an optimization to abort as early as possible to skip unnecessary\n    // updates.\n    //\n    // However, changeLog entries that are destined to be \"skipped\" are\n    // nonetheless kept for the purpose of tracking backfillingColumnVersions.\n    this.#logTableWideOpStmt = db.prepare(/*sql*/ `\n      INSERT OR REPLACE INTO \"_zero.changeLog2\" \n        (stateVersion, pos, \"table\", rowKey, op) \n        VALUES (@version, -1, @table, @version, @op)\n    `);\n\n    this.#getRowOpStmt = db.prepare(/*sql*/ `\n      SELECT * FROM \"_zero.changeLog2\" WHERE \"table\" = ? AND \"rowKey\" = JSON(?)\n    `);\n  }\n\n  /**\n   *\n   * @param backfilled The backfilling columns for which values were set. Note\n   *   that an empty list and the `undefined` value mean different things;\n   *   * An empty list indicates that a backfill is in progress but no\n   *     backfilling values were set. In this case, existing\n   *     backfillingColumnVersions are preserved.\n   *   * `undefined` indicates that there are no columns being backfilled.\n   *     In this case, any vestigial `backfillingColumnVersions` value\n   *     is cleared.\n   */\n  logSetOp(\n    version: LexiVersion,\n    pos: number,\n    table: string,\n    row: LiteRowKey,\n    backfilled: string[] | undefined,\n  ): string {\n    return this.#logRowOp(version, pos, table, row, SET_OP, backfilled);\n  }\n\n  logDeleteOp(\n    version: LexiVersion,\n    pos: number,\n    table: string,\n    row: LiteRowKey,\n  ): string {\n    // Note: For delete ops, it is always safe to clear the\n    //       backfillingColumnVersions because the backfill algorithm\n    //       understands that deletes apply to the whole row.\n    return this.#logRowOp(version, pos, table, row, DEL_OP, undefined);\n  }\n\n  getLatestRowOp(table: string, row: LiteRowKey) {\n    const rowKey = stringify(normalizedKeyOrder(row));\n    const result = this.#getRowOpStmt.get(table, rowKey);\n    return result === undefined\n      ? undefined\n      : v.parse(result, rawChangeLogEntrySchema, 'passthrough');\n  }\n\n  #logRowOp(\n    version: LexiVersion,\n    pos: number,\n    table: string,\n    row: LiteRowKey,\n    op: string,\n    backfilled: string[] | undefined,\n  ): string {\n    const rowKey = stringify(normalizedKeyOrder(row));\n    if (backfilled === undefined) {\n      this.#logRowOpStmt.run({version, pos, table, rowKey, op});\n    } else {\n      const versions: Record<string, string> = {};\n      for (const col of backfilled) {\n        versions[col] = version;\n      }\n      this.#logRowOpWithBackfillStmt.run({\n        version,\n        pos,\n        table,\n        rowKey,\n        op,\n        backfillingColumnVersions: JSON.stringify(versions),\n      });\n    }\n    return rowKey;\n  }\n\n  logTruncateOp(version: LexiVersion, table: string) {\n    this.#logTableWideOpStmt.run({version, table, op: TRUNCATE_OP});\n  }\n\n  logResetOp(version: LexiVersion, table: string) {\n    this.#logTableWideOpStmt.run({version, table, op: RESET_OP});\n  }\n}\n"],"mappings":";;;AAgDA,IAAa,0BAwBX;;;;;;;;;;;;;;;;;;AAmBF,IAAa,uBAAuB,eACjC,OAAO;CACN,cAAc,eAAE,QAAQ;CACxB,OAAO,eAAE,QAAQ;CACjB,QAAQ,eAAE,QAAQ;CAClB,IAAI,aAAA,KAAA,KAAA,KAAA,IAAqD;CAC1D,CAAC,CACD,KAAI,SAAQ;CACX,GAAG;CAEH,QACE,IAAI,OAAO,OAAO,IAAI,OAAO,MACzB,OACA,MAAQ,QAAM,IAAI,OAAO,EAAE,iBAAiB;CACnD,EAAE;AAIL,IAAM,0BAA0B,eAAE,OAAO;CACvC,cAAc,eAAE,QAAQ;CACxB,OAAO,eAAE,QAAQ;CACjB,QAAQ,eAAE,QAAQ;CAClB,IAAI,aAAA,KAAA,KAAA,KAAA,IAAqD;CACzD,2BAA2B,eACxB,QAAQ,CACR,KAAI,QAAO,eAAE,OAAO,eAAE,QAAQ,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC;CAC3D,CAAC;AAIF,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CAEA,YAAY,IAAc;AACxB,QAAA,eAAqB,GAAG,QAAgB;;;;MAItC;AAEF,QAAA,2BAAiC,GAAG,QAAgB;;;;;;;;;;;MAWlD;AASF,QAAA,qBAA2B,GAAG,QAAgB;;;;MAI5C;AAEF,QAAA,eAAqB,GAAG,QAAgB;;MAEtC;;;;;;;;;;;;;CAcJ,SACE,SACA,KACA,OACA,KACA,YACQ;AACR,SAAO,MAAA,SAAe,SAAS,KAAK,OAAO,KAAA,KAAa,WAAW;;CAGrE,YACE,SACA,KACA,OACA,KACQ;AAIR,SAAO,MAAA,SAAe,SAAS,KAAK,OAAO,KAAA,KAAa,KAAA,EAAU;;CAGpE,eAAe,OAAe,KAAiB;EAC7C,MAAM,SAAS,UAAU,mBAAmB,IAAI,CAAC;EACjD,MAAM,SAAS,MAAA,aAAmB,IAAI,OAAO,OAAO;AACpD,SAAO,WAAW,KAAA,IACd,KAAA,IACA,MAAQ,QAAQ,yBAAyB,cAAc;;CAG7D,UACE,SACA,KACA,OACA,KACA,IACA,YACQ;EACR,MAAM,SAAS,UAAU,mBAAmB,IAAI,CAAC;AACjD,MAAI,eAAe,KAAA,EACjB,OAAA,aAAmB,IAAI;GAAC;GAAS;GAAK;GAAO;GAAQ;GAAG,CAAC;OACpD;GACL,MAAM,WAAmC,EAAE;AAC3C,QAAK,MAAM,OAAO,WAChB,UAAS,OAAO;AAElB,SAAA,yBAA+B,IAAI;IACjC;IACA;IACA;IACA;IACA;IACA,2BAA2B,KAAK,UAAU,SAAS;IACpD,CAAC;;AAEJ,SAAO;;CAGT,cAAc,SAAsB,OAAe;AACjD,QAAA,mBAAyB,IAAI;GAAC;GAAS;GAAO,IAAA;GAAgB,CAAC;;CAGjE,WAAW,SAAsB,OAAe;AAC9C,QAAA,mBAAyB,IAAI;GAAC;GAAS;GAAO,IAAA;GAAa,CAAC"}
+{"version":3,"file":"change-log.js","names":["#logRowOpStmt","#logRowOpWithBackfillStmt","#logTableWideOpStmt","#getRowOpStmt","#logRowOp"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/change-log.ts"],"sourcesContent":["import {\n  jsonObjectSchema,\n  parse,\n  stringify,\n} from '../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport type {LexiVersion} from '../../../types/lexi-version.ts';\nimport type {LiteRowKey} from '../../../types/lite.ts';\nimport {normalizedKeyOrder} from '../../../types/row-key.ts';\n\n/**\n * The Change Log tracks the last operation (set or delete) for each row in the\n * data base, ordered by state version; in other words, a cross-table\n * index of row changes ordered by version. This facilitates a minimal \"diff\"\n * of row changes needed to advance a pipeline from one state version to another.\n *\n * The Change Log stores identifiers only, i.e. it does not store contents.\n * A database snapshot at the previous version can be used to query a row's\n * old contents, if any, and the current snapshot can be used to query a row's\n * new contents. (In the common case, the new contents will have just been applied\n * and thus has a high likelihood of being in the SQLite cache.)\n *\n * There are two table-wide operations:\n * - `t` corresponds to the postgres `TRUNCATE` operation\n * - `r` represents any schema (i.e. column) change\n *\n * For both operations, the corresponding row changes are not explicitly included\n * in the change log. The consumer has the option of simulating them be reading\n * from pre- and post- snapshots, or resetting their state entirely with the current\n * snapshot.\n *\n * To achieve the desired ordering semantics when processing tables that have been\n * truncated, reset, and modified, the \"rowKey\" is set to `null` for resets and\n * the empty string `\"\"` for truncates. This means that resets will be encountered\n * before truncates, which will be processed before any subsequent row changes.\n *\n * This ordering is chosen because resets are currently the more \"destructive\" op\n * and result in aborting the processing (and starting from scratch); doing this\n * earlier reduces wasted work.\n */\n\nexport const SET_OP = 's';\nexport const DEL_OP = 'd';\nexport const TRUNCATE_OP = 't';\nexport const RESET_OP = 'r';\n\n// Exported for testing (and migrations)\nexport const CREATE_CHANGELOG_SCHEMA =\n  // stateVersion : a.k.a. row version\n  // pos          : order in which to process the change (within the version)\n  // table        : The table associated with the change\n  // rowKey       : JSON row key for a row change. For table-wide changes RESET\n  //                and TRUNCATE, there is no associated row; instead, `pos` is\n  //                set to -1 and the rowKey is set to the stateVersion,\n  //                guaranteeing when attempting to process the transaction,\n  //                the pipeline is reset (and the change log traversal\n  //                aborted).\n  // op           : 's' for set (insert/update)\n  //              : 'd' for delete\n  //              : 'r' for table reset (schema change)\n  //              : 't' for table truncation (which also resets the pipeline)\n  // backfillingColumnVersions\n  //              : A JSON mapping from column name to stateVersion tracked\n  //                for replicated writes of columns that are being backfilled.\n  //                This is used to prevent backfill data, which is at a\n  //                fixed snapshot/version outside of the replication stream,\n  //                from overwriting newer column values.\n  //\n  // Naming note: To maintain compatibility between a new replication-manager\n  // and old view-syncers, the previous _zero.changeLog table is preserved\n  // and its replacement given a new name \"changeLog2\".\n  `\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    \"backfillingColumnVersions\" TEXT DEFAULT '{}',\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n  `;\n\n/**\n * Contains the changeLog fields relevant for computing the diff between\n * two snapshots of a replica. The `pos` and `backfillingColumnVersions`\n * fields are excluded, though the query should be ordered by\n * `<stateVersion, pos>`.\n */\nexport const changeLogEntrySchema = v\n  .object({\n    stateVersion: v.string(),\n    table: v.string(),\n    rowKey: v.string(),\n    op: v.literalUnion(SET_OP, DEL_OP, TRUNCATE_OP, RESET_OP),\n  })\n  .map(val => ({\n    ...val,\n    // Note: sets the rowKey to `null` for table-wide ops / resets\n    rowKey:\n      val.op === 't' || val.op === 'r'\n        ? null\n        : v.parse(parse(val.rowKey), jsonObjectSchema),\n  }));\n\nexport type ChangeLogEntry = v.Infer<typeof changeLogEntrySchema>;\n\nconst rawChangeLogEntrySchema = v.object({\n  stateVersion: v.string(),\n  table: v.string(),\n  rowKey: v.string(),\n  op: v.literalUnion(SET_OP, DEL_OP, TRUNCATE_OP, RESET_OP),\n  backfillingColumnVersions: v\n    .string()\n    .map(val => v.record(v.string()).parse(JSON.parse(val))),\n});\n\nexport type RawChangeLogEntry = v.Infer<typeof rawChangeLogEntrySchema>;\n\nexport class ChangeLog {\n  readonly #logRowOpStmt: Statement;\n  readonly #logRowOpWithBackfillStmt: Statement;\n  readonly #logTableWideOpStmt;\n  readonly #getRowOpStmt: Statement;\n\n  constructor(db: Database) {\n    this.#logRowOpStmt = db.prepare(/*sql*/ `\n      INSERT OR REPLACE INTO \"_zero.changeLog2\" \n        (stateVersion, pos, \"table\", rowKey, op)\n        VALUES (@version, @pos, @table, JSON(@rowKey), @op)\n    `);\n\n    this.#logRowOpWithBackfillStmt = db.prepare(/*sql*/ `\n      INSERT INTO \"_zero.changeLog2\" \n        (stateVersion, pos, \"table\", rowKey, op, backfillingColumnVersions)\n        VALUES (@version, @pos, @table, JSON(@rowKey), @op, \n                JSON(@backfillingColumnVersions))\n        ON CONFLICT (\"table\", rowKey) DO UPDATE \n                   SET stateVersion = excluded.stateVersion,\n                                pos = excluded.pos,\n                                 op = excluded.op,\n          backfillingColumnVersions = json_patch(\n          backfillingColumnVersions, excluded.backfillingColumnVersions)\n    `);\n\n    // Because table-wide ops result in aborting an incremental update\n    // and rehydrating all queries at \"head\", they are assigned pos = -1\n    // as an optimization to abort as early as possible to skip unnecessary\n    // updates.\n    //\n    // However, changeLog entries that are destined to be \"skipped\" are\n    // nonetheless kept for the purpose of tracking backfillingColumnVersions.\n    this.#logTableWideOpStmt = db.prepare(/*sql*/ `\n      INSERT OR REPLACE INTO \"_zero.changeLog2\" \n        (stateVersion, pos, \"table\", rowKey, op) \n        VALUES (@version, -1, @table, @version, @op)\n    `);\n\n    // oxlint-disable-next-line zero/no-select-star -- Local SQLite replica query; not run through pg prepared statements.\n    this.#getRowOpStmt = db.prepare(/*sql*/ `\n      SELECT * FROM \"_zero.changeLog2\" WHERE \"table\" = ? AND \"rowKey\" = JSON(?)\n    `);\n  }\n\n  /**\n   *\n   * @param backfilled The backfilling columns for which values were set. Note\n   *   that an empty list and the `undefined` value mean different things;\n   *   * An empty list indicates that a backfill is in progress but no\n   *     backfilling values were set. In this case, existing\n   *     backfillingColumnVersions are preserved.\n   *   * `undefined` indicates that there are no columns being backfilled.\n   *     In this case, any vestigial `backfillingColumnVersions` value\n   *     is cleared.\n   */\n  logSetOp(\n    version: LexiVersion,\n    pos: number,\n    table: string,\n    row: LiteRowKey,\n    backfilled: string[] | undefined,\n  ): string {\n    return this.#logRowOp(version, pos, table, row, SET_OP, backfilled);\n  }\n\n  logDeleteOp(\n    version: LexiVersion,\n    pos: number,\n    table: string,\n    row: LiteRowKey,\n  ): string {\n    // Note: For delete ops, it is always safe to clear the\n    //       backfillingColumnVersions because the backfill algorithm\n    //       understands that deletes apply to the whole row.\n    return this.#logRowOp(version, pos, table, row, DEL_OP, undefined);\n  }\n\n  getLatestRowOp(table: string, row: LiteRowKey) {\n    const rowKey = stringify(normalizedKeyOrder(row));\n    const result = this.#getRowOpStmt.get(table, rowKey);\n    return result === undefined\n      ? undefined\n      : v.parse(result, rawChangeLogEntrySchema, 'passthrough');\n  }\n\n  #logRowOp(\n    version: LexiVersion,\n    pos: number,\n    table: string,\n    row: LiteRowKey,\n    op: string,\n    backfilled: string[] | undefined,\n  ): string {\n    const rowKey = stringify(normalizedKeyOrder(row));\n    if (backfilled === undefined) {\n      this.#logRowOpStmt.run({version, pos, table, rowKey, op});\n    } else {\n      const versions: Record<string, string> = {};\n      for (const col of backfilled) {\n        versions[col] = version;\n      }\n      this.#logRowOpWithBackfillStmt.run({\n        version,\n        pos,\n        table,\n        rowKey,\n        op,\n        backfillingColumnVersions: JSON.stringify(versions),\n      });\n    }\n    return rowKey;\n  }\n\n  logTruncateOp(version: LexiVersion, table: string) {\n    this.#logTableWideOpStmt.run({version, table, op: TRUNCATE_OP});\n  }\n\n  logResetOp(version: LexiVersion, table: string) {\n    this.#logTableWideOpStmt.run({version, table, op: RESET_OP});\n  }\n}\n"],"mappings":";;;AAgDA,IAAa,0BAwBX;;;;;;;;;;;;;;;;;;AAmBF,IAAa,uBAAuB,eACjC,OAAO;CACN,cAAc,eAAE,QAAQ;CACxB,OAAO,eAAE,QAAQ;CACjB,QAAQ,eAAE,QAAQ;CAClB,IAAI,aAAA,KAAA,KAAA,KAAA,IAAqD;CAC1D,CAAC,CACD,KAAI,SAAQ;CACX,GAAG;CAEH,QACE,IAAI,OAAO,OAAO,IAAI,OAAO,MACzB,OACA,MAAQ,QAAM,IAAI,OAAO,EAAE,iBAAiB;CACnD,EAAE;AAIL,IAAM,0BAA0B,eAAE,OAAO;CACvC,cAAc,eAAE,QAAQ;CACxB,OAAO,eAAE,QAAQ;CACjB,QAAQ,eAAE,QAAQ;CAClB,IAAI,aAAA,KAAA,KAAA,KAAA,IAAqD;CACzD,2BAA2B,eACxB,QAAQ,CACR,KAAI,QAAO,eAAE,OAAO,eAAE,QAAQ,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC;CAC3D,CAAC;AAIF,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CAEA,YAAY,IAAc;AACxB,QAAA,eAAqB,GAAG,QAAgB;;;;MAItC;AAEF,QAAA,2BAAiC,GAAG,QAAgB;;;;;;;;;;;MAWlD;AASF,QAAA,qBAA2B,GAAG,QAAgB;;;;MAI5C;AAGF,QAAA,eAAqB,GAAG,QAAgB;;MAEtC;;;;;;;;;;;;;CAcJ,SACE,SACA,KACA,OACA,KACA,YACQ;AACR,SAAO,MAAA,SAAe,SAAS,KAAK,OAAO,KAAA,KAAa,WAAW;;CAGrE,YACE,SACA,KACA,OACA,KACQ;AAIR,SAAO,MAAA,SAAe,SAAS,KAAK,OAAO,KAAA,KAAa,KAAA,EAAU;;CAGpE,eAAe,OAAe,KAAiB;EAC7C,MAAM,SAAS,UAAU,mBAAmB,IAAI,CAAC;EACjD,MAAM,SAAS,MAAA,aAAmB,IAAI,OAAO,OAAO;AACpD,SAAO,WAAW,KAAA,IACd,KAAA,IACA,MAAQ,QAAQ,yBAAyB,cAAc;;CAG7D,UACE,SACA,KACA,OACA,KACA,IACA,YACQ;EACR,MAAM,SAAS,UAAU,mBAAmB,IAAI,CAAC;AACjD,MAAI,eAAe,KAAA,EACjB,OAAA,aAAmB,IAAI;GAAC;GAAS;GAAK;GAAO;GAAQ;GAAG,CAAC;OACpD;GACL,MAAM,WAAmC,EAAE;AAC3C,QAAK,MAAM,OAAO,WAChB,UAAS,OAAO;AAElB,SAAA,yBAA+B,IAAI;IACjC;IACA;IACA;IACA;IACA;IACA,2BAA2B,KAAK,UAAU,SAAS;IACpD,CAAC;;AAEJ,SAAO;;CAGT,cAAc,SAAsB,OAAe;AACjD,QAAA,mBAAyB,IAAI;GAAC;GAAS;GAAO,IAAA;GAAgB,CAAC;;CAGjE,WAAW,SAAsB,OAAe;AAC9C,QAAA,mBAAyB,IAAI;GAAC;GAAS;GAAO,IAAA;GAAa,CAAC"}

package/out/zero-cache/src/services/view-syncer/cvr-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cvr-store.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAiBjD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AAMnF,OAAO,EAAC,sBAAsB,EAAC,MAAM,iCAAiC,CAAC;AACvE,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AAEvE,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAQ,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAC/D,OAAO,KAAK,EAAC,GAAG,EAAE,WAAW,EAAC,MAAM,UAAU,CAAC;AAE/C,OAAO,EAKL,KAAK,OAAO,EACb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAEL,KAAK,YAAY,EAGjB,KAAK,UAAU,EAGf,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,WAAW,EAEhB,KAAK,KAAK,EACV,KAAK,SAAS,EAGf,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,KAAK,QAAQ,EAGd,MAAM,gBAAgB,CAAC;AAExB,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAqGF,qBAAa,QAAQ;;gBAmCjB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,EACjC,qBAAqB,SAA2B,EAChD,eAAe,SAAoB,EACnC,yBAAyB,SAAM,EAAE,qBAAqB;IACtD,YAAY,oBAAa;IAmC3B,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAyN3D,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAIvD,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI;IAIlC;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,EAAE,KAAK,GAAG,IAAI;IAI7B;;;;OAIG;IACH,YAAY,CAAC,GAAG,GAAG,EAAE,KAAK,EAAE;IAM5B;;;;OAIG;IACG,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO3E;;;;;OAKG;IACG,WAAW,IAAI,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAelD,WAAW,CAAC,EACV,OAAO,EACP,cAAc,EACd,UAAU,EACV,YAAY,EACZ,SAAS,EACT,QAAQ,GACT,EAAE,IAAI,CACL,WAAW,EACT,SAAS,GACT,gBAAgB,GAChB,YAAY,GACZ,cAAc,GACd,WAAW,GACX,UAAU,CACb,GAAG,IAAI;IAqBR,kBAAkB,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,GAAG,IAAI;IASrE,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAelC,WAAW,CAAC,KAAK,EAAE,WAAW;IAc9B,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IAIjE,YAAY,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAYxC,YAAY,CAAC,QAAQ,EAAE,MAAM;IAU7B,eAAe,CACb,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EACnB,MAAM,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EACpB,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,QAAQ,GAAG,SAAS,EACnC,GAAG,EAAE,MAAM,GACV,IAAI;IAkBP,iBAAiB,CACf,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,kBAAkB,EAChC,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,UAAU,EACnB,kBAAkB,GAAE,MAAM,EAAO,GAChC,cAAc,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC;IAUvC,oBAAoB,CACxB,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,kBAAkB,EAChC,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,cAAc,EAAE,CAAC;IAwe5B,IAAI,QAAQ,IAAI,MAAM,CAErB;IAEK,KAAK,CACT,EAAE,EAAE,UAAU,EACd,sBAAsB,EAAE,UAAU,EAClC,GAAG,EAAE,WAAW,EAChB,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAkChC,iBAAiB,IAAI,OAAO;IAI5B,qDAAqD;IACrD,OAAO,CAAC,EAAE,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhC,cAAc,CAClB,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,QAAQ,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,EAAE,CAAC;CAsC9B;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAChC,EAAE,EAAE,mBAAmB,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,sBAAsB,EAAE,UAAU,GACjC,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,qBAAa,mBAAoB,SAAQ,sBAAsB;gBACjD,OAAO,EAAE,MAAM;CAU5B;AAED,qBAAa,+BAAgC,SAAQ,sBAAsB;IACzE,QAAQ,CAAC,IAAI,qCAAqC;gBAEtC,eAAe,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM;CAU3D;AAED,qBAAa,cAAe,SAAQ,sBAAsB;IACxD,QAAQ,CAAC,IAAI,oBAAoB;gBAG/B,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,eAAe,EAAE,MAAM;CAe1B;AAED,qBAAa,wBAAyB,SAAQ,sBAAsB;IAClE,QAAQ,CAAC,IAAI,8BAA8B;gBAE/B,KAAK,EAAE,OAAO;CAW3B;AAED,qBAAa,sBAAuB,SAAQ,KAAK;IAC/C,QAAQ,CAAC,IAAI,4BAA4B;IACzC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAExB,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;CAK3D"}
+{"version":3,"file":"cvr-store.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAiBjD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AAMnF,OAAO,EAAC,sBAAsB,EAAC,MAAM,iCAAiC,CAAC;AACvE,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AAEvE,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAQ,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAC/D,OAAO,KAAK,EAAC,GAAG,EAAE,WAAW,EAAC,MAAM,UAAU,CAAC;AAE/C,OAAO,EAKL,KAAK,OAAO,EACb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAEL,KAAK,YAAY,EAGjB,KAAK,UAAU,EAGf,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,WAAW,EAEhB,KAAK,KAAK,EACV,KAAK,SAAS,EAGf,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,KAAK,QAAQ,EAGd,MAAM,gBAAgB,CAAC;AAExB,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAqGF,qBAAa,QAAQ;;gBAmCjB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,EACjC,qBAAqB,SAA2B,EAChD,eAAe,SAAoB,EACnC,yBAAyB,SAAM,EAAE,qBAAqB;IACtD,YAAY,oBAAa;IAmC3B,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAqO3D,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAIvD,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI;IAIlC;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,EAAE,KAAK,GAAG,IAAI;IAI7B;;;;OAIG;IACH,YAAY,CAAC,GAAG,GAAG,EAAE,KAAK,EAAE;IAM5B;;;;OAIG;IACG,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO3E;;;;;OAKG;IACG,WAAW,IAAI,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAelD,WAAW,CAAC,EACV,OAAO,EACP,cAAc,EACd,UAAU,EACV,YAAY,EACZ,SAAS,EACT,QAAQ,GACT,EAAE,IAAI,CACL,WAAW,EACT,SAAS,GACT,gBAAgB,GAChB,YAAY,GACZ,cAAc,GACd,WAAW,GACX,UAAU,CACb,GAAG,IAAI;IAqBR,kBAAkB,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,GAAG,IAAI;IASrE,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAelC,WAAW,CAAC,KAAK,EAAE,WAAW;IAc9B,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IAIjE,YAAY,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAYxC,YAAY,CAAC,QAAQ,EAAE,MAAM;IAU7B,eAAe,CACb,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EACnB,MAAM,EAAE;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,EACpB,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,QAAQ,GAAG,SAAS,EACnC,GAAG,EAAE,MAAM,GACV,IAAI;IAkBP,iBAAiB,CACf,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,kBAAkB,EAChC,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,UAAU,EACnB,kBAAkB,GAAE,MAAM,EAAO,GAChC,cAAc,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC;IAUvC,oBAAoB,CACxB,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,kBAAkB,EAChC,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,cAAc,EAAE,CAAC;IAgf5B,IAAI,QAAQ,IAAI,MAAM,CAErB;IAEK,KAAK,CACT,EAAE,EAAE,UAAU,EACd,sBAAsB,EAAE,UAAU,EAClC,GAAG,EAAE,WAAW,EAChB,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAkChC,iBAAiB,IAAI,OAAO;IAI5B,qDAAqD;IACrD,OAAO,CAAC,EAAE,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhC,cAAc,CAClB,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,QAAQ,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,EAAE,CAAC;CAsC9B;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAChC,EAAE,EAAE,mBAAmB,EACvB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,sBAAsB,EAAE,UAAU,GACjC,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,qBAAa,mBAAoB,SAAQ,sBAAsB;gBACjD,OAAO,EAAE,MAAM;CAU5B;AAED,qBAAa,+BAAgC,SAAQ,sBAAsB;IACzE,QAAQ,CAAC,IAAI,qCAAqC;gBAEtC,eAAe,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM;CAU3D;AAED,qBAAa,cAAe,SAAQ,sBAAsB;IACxD,QAAQ,CAAC,IAAI,oBAAoB;gBAG/B,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,eAAe,EAAE,MAAM;CAe1B;AAED,qBAAa,wBAAyB,SAAQ,sBAAsB;IAClE,QAAQ,CAAC,IAAI,8BAA8B;gBAE/B,KAAK,EAAE,OAAO;CAW3B;AAED,qBAAa,sBAAuB,SAAQ,KAAK;IAC/C,QAAQ,CAAC,IAAI,4BAA4B;IACzC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAExB,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;CAK3D"}

package/out/zero-cache/src/services/view-syncer/cvr-store.js

@@ -164,7 +164,19 @@ var CVRStore = class {
             WHERE cvr."clientGroupID" = ${id}`,
 				tx`SELECT "clientID" FROM ${this.#cvr("clients")}
            WHERE "clientGroupID" = ${id}`,
-				tx`SELECT * FROM ${this.#cvr("queries")}
+				tx`SELECT
+            "clientGroupID",
+            "queryHash",
+            "clientAST",
+            "queryName",
+            "queryArgs",
+            "patchVersion",
+            "transformationHash",
+            "transformationVersion",
+            "internal",
+            "deleted",
+            "rowSetSignature"
+          FROM ${this.#cvr("queries")}
           WHERE "clientGroupID" = ${id} AND deleted IS DISTINCT FROM true`,
 				tx`SELECT
           "clientGroupID",
@@ -378,7 +390,15 @@ var CVRStore = class {
 		try {
 			await reader.processReadTask((tx) => checkVersion(tx, this.#schema, this.#id, current));
 			const [allDesires, queryRows] = await reader.processReadTask((tx) => Promise.all([tx`
-      SELECT * FROM ${this.#cvr("desires")}
+      SELECT
+        "clientGroupID",
+        "clientID",
+        "queryHash",
+        "patchVersion",
+        "deleted",
+        "ttl",
+        "inactivatedAt"
+      FROM ${this.#cvr("desires")}
         WHERE "clientGroupID" = ${this.#id}
         AND "patchVersion" > ${start}
         AND "patchVersion" <= ${end}`, tx`