@rocicorp/zero 1.11.0-canary.6 → 1.11.0-canary.7

package/out/zero-cache/src/server/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  // Startup-time client used for for change-streamer initialization\n  // and handoff / takeover. Steady-state clients are managed by the\n  // change-streamer (Storer) implementation.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n\n  // When restoring from litestream, acquire a lock to prevent change-log\n  // purges. This ensures that (this) change-streamer will be able to resume\n  // from the backup.\n  let purgeLock =\n    pgChangeLogEnabled && litestream.backupURL && litestream.executable\n      ? await new PurgeLocker(lc, shard, changeDB).acquire()\n      : null;\n  const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  let waitForFirstBackupBeforeServing = false;\n  for (const first of [true, false]) {\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n      } =\n        upstream.type === 'pg'\n          ? await initializePostgresChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              {\n                ...initialSync,\n                replicationSlotFailover: upstream.pgReplicationSlotFailover,\n                installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n              },\n              context,\n              replicationLag.reportIntervalMs,\n              restoreOptions,\n              {backupV5: litestream.backupUsingV5},\n              purgeLock,\n              upstream.pgStreamInboundTimeoutMs,\n            )\n          : await initializeCustomChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              context,\n              restoreOptions,\n            );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogEnabled,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      break;\n    } catch (e) {\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        // Release the purge lock before retrying. This is safe because the\n        // purge lock exists to preserve change-log entries so the new\n        // change-streamer can resume from the backup replica's watermark.\n        // An AutoResetSignal means we cant resume from the backup replica\n        // (e.g. its replication slot is gone), so the change-log entries the lock\n        // was protecting are no longer needed. The retry performs a fresh\n        // initial sync with a new replication slot, independent of the old\n        // change-log. Releasing is also necessary to avoid a\n        // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n        // CREATE_REPLICATION_SLOT waits for all older transactions to\n        // finish, including this lock's open transaction.\n        await purgeLock?.release();\n        purgeLock = null;\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  const processes = new ProcessManager(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  if (backupURL) {\n    lc.info?.('setting up backup to', backupURL);\n    litestream.backupURL = backupURL;\n    const {promise: backupStarted, resolve} = resolver();\n\n    // Start a backup replicator and corresponding litestream backup process.\n    const backupReplicator = processes\n      .addWorker(\n        childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n        'supporting',\n        'backup-replicator',\n      )\n      // Wait for the replicator's first message (i.e. \"ready\") before starting\n      // litestream backup in order to avoid contending on the sqlite lock\n      // when the replicator first prepares the db file.\n      .once('message', () => {\n        processes.addSubprocess(\n          startReplicaBackupProcess(lc, litestream, replica.file),\n          'supporting',\n          'litestream',\n        );\n        resolve();\n      });\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n    await backupStarted;\n  }\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n  });\n\n  let readinessGate = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    readinessGate = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n    });\n  }\n\n  const getProfileWorker =\n    profileSubWorkers.length > 0\n      ? () => Promise.resolve(broadcastWorker(profileSubWorkers))\n      : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      startupDelayMs,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  try {\n    await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'change-streamer');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        await publishCriticalEvent(\n          lc,\n          replicationStatusError(lc, 'Initializing', e),\n        );\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,sBAAsB,WAAW,aAAa,WAAW,aACrD,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GAClC,6BAA6B,SAAS;EACxC,GACA,SACA,eAAe,kBACf,gBACA,EAAC,UAAU,WAAW,cAAa,GACnC,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;EACvD,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAEJ,wBACE,sBAAsB,2BAClB;IACE,aAAa,QAAQ;IACrB,gBAAgB;IAChB,aAAa;IACb,eAAe;GACjB,IACA,KAAA;GAIN,sBACE,wBAAwB,UACpB;IACE,aAAa;IACb,iBAAiB;IACjB,aAAa;GACf,IACA,KAAA;EACR,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,MAAM,oBAA8B,CAAC;CACrC,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,MAAM,mBAAmB,UACtB,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;EACA,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,MAAM,mBACJ,kBAAkB,SAAS,UACjB,QAAQ,QAAQ,gBAAgB,iBAAiB,CAAC,IACxD,KAAA;CAEN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EACA;EACA;EACA;EACA;CACF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  // Startup-time client used for for change-streamer initialization\n  // and handoff / takeover. Steady-state clients are managed by the\n  // change-streamer (Storer) implementation.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n\n  // When restoring from litestream, acquire a lock to prevent change-log\n  // purges. This ensures that (this) change-streamer will be able to resume\n  // from the backup.\n  let purgeLock =\n    pgChangeLogEnabled && litestream.backupURL && litestream.executable\n      ? await new PurgeLocker(lc, shard, changeDB).acquire()\n      : null;\n  const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  let waitForFirstBackupBeforeServing = false;\n  for (const first of [true, false]) {\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n      } =\n        upstream.type === 'pg'\n          ? await initializePostgresChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              {\n                ...initialSync,\n                replicationSlotFailover: upstream.pgReplicationSlotFailover,\n                installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n              },\n              context,\n              replicationLag.reportIntervalMs,\n              restoreOptions,\n              {backupV5: litestream.backupUsingV5},\n              purgeLock,\n              upstream.pgStreamInboundTimeoutMs,\n            )\n          : await initializeCustomChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              context,\n              restoreOptions,\n            );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogEnabled,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      break;\n    } catch (e) {\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        // Release the purge lock before retrying. This is safe because the\n        // purge lock exists to preserve change-log entries so the new\n        // change-streamer can resume from the backup replica's watermark.\n        // An AutoResetSignal means we cant resume from the backup replica\n        // (e.g. its replication slot is gone), so the change-log entries the lock\n        // was protecting are no longer needed. The retry performs a fresh\n        // initial sync with a new replication slot, independent of the old\n        // change-log. Releasing is also necessary to avoid a\n        // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n        // CREATE_REPLICATION_SLOT waits for all older transactions to\n        // finish, including this lock's open transaction.\n        await purgeLock?.release();\n        purgeLock = null;\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  const processes = new ProcessManager(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  if (backupURL) {\n    lc.info?.('setting up backup to', backupURL);\n    litestream.backupURL = backupURL;\n    const {promise: backupStarted, resolve} = resolver();\n\n    // Start a backup replicator and corresponding litestream backup process.\n    const backupReplicator = processes\n      .addWorker(\n        childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n        'supporting',\n        'backup-replicator',\n      )\n      // Wait for the replicator's first message (i.e. \"ready\") before starting\n      // litestream backup in order to avoid contending on the sqlite lock\n      // when the replicator first prepares the db file.\n      .once('message', () => {\n        processes.addSubprocess(\n          startReplicaBackupProcess(lc, litestream, replica.file),\n          'supporting',\n          'litestream',\n        );\n        resolve();\n      });\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n    await backupStarted;\n  }\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n  });\n\n  let readinessGate = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    readinessGate = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n    });\n  }\n\n  // Create the broadcast facade once: each broadcastWorker() adds permanent\n  // 'message' forwarders to every sub worker, so creating one per /profz\n  // request would leak a forwarder per request.\n  const profileWorker =\n    profileSubWorkers.length > 0\n      ? broadcastWorker(profileSubWorkers)\n      : undefined;\n  const getProfileWorker = profileWorker\n    ? () => Promise.resolve(profileWorker)\n    : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      startupDelayMs,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  try {\n    await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'change-streamer');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        await publishCriticalEvent(\n          lc,\n          replicationStatusError(lc, 'Initializing', e),\n        );\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,sBAAsB,WAAW,aAAa,WAAW,aACrD,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GAClC,6BAA6B,SAAS;EACxC,GACA,SACA,eAAe,kBACf,gBACA,EAAC,UAAU,WAAW,cAAa,GACnC,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;EACvD,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAEJ,wBACE,sBAAsB,2BAClB;IACE,aAAa,QAAQ;IACrB,gBAAgB;IAChB,aAAa;IACb,eAAe;GACjB,IACA,KAAA;GAIN,sBACE,wBAAwB,UACpB;IACE,aAAa;IACb,iBAAiB;IACjB,aAAa;GACf,IACA,KAAA;EACR,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,MAAM,oBAA8B,CAAC;CACrC,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,MAAM,mBAAmB,UACtB,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;EACA,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAKA,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EACA;EACA;EACA;EACA,kBAZqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAWF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-manager.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,EAAmB,KAAK,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAKzE,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EAEnB,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,uBAAuB,EACvB,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAMxC;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,eAAe,GAAG,iBAAiB,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,KAAK,gBAAgB,GAAG,CACtB,GAAG,EAAE,eAAe,KACjB,cAAc,CAAC,eAAe,CAAC,CAAC;AAuBrC;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,UAAU,EAAE,QAAQ;;IAgC1D,YACE,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,GAAE,UAA6B,EACzC,YAAY,SAA0B,EACtC,YAAY,SAA0B,EACtC,oBAAoB,SAAyB,EAU9C;IAED,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,QAU5D;IAqPD;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI,CA2M3C;IAED,MAAM,IAAI,IAAI,CAGb;CACF;AAED,uBAAe,mBAAoB,SAAQ,KAAK;IAC9C,YAAY,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAM5D;CACF;AA6BD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,mBAAmB;IACjE,QAAQ,CAAC,IAAI,gCAAgC;IAE7C,YAAY,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAE5D;CACF"}
+{"version":3,"file":"backfill-manager.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,EAAmB,KAAK,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAKzE,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EAEnB,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,uBAAuB,EACvB,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAMxC;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,eAAe,GAAG,iBAAiB,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,KAAK,gBAAgB,GAAG,CACtB,GAAG,EAAE,eAAe,KACjB,cAAc,CAAC,eAAe,CAAC,CAAC;AAuBrC;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,UAAU,EAAE,QAAQ;;IAmC1D,YACE,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,GAAE,UAA6B,EACzC,YAAY,SAA0B,EACtC,YAAY,SAA0B,EACtC,oBAAoB,SAAyB,EAU9C;IAED,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,QAU5D;IAiQD;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI,CA2M3C;IAED,MAAM,IAAI,IAAI,CAab;CACF;AAED,uBAAe,mBAAoB,SAAQ,KAAK;IAC9C,YAAY,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAM5D;CACF;AA6BD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,mBAAmB;IACjE,QAAQ,CAAC,IAAI,gCAAgC;IAE7C,YAAY,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAE5D;CACF"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.js

@@ -57,6 +57,8 @@ var BackfillManager = class {
 	/** The watermark of the current transaction in the change stream. */
 	#currentTxWatermark = null;
 	#commitThresholdBytes;
+	/** Set when the change stream is canceled. No further backfills are run. */
+	#canceled = false;
 	constructor(lc, changeStreamer, backfillStreamer, jsonFormat = "s", minBackoffMs = MIN_BACKOFF_INTERVAL_MS, maxBackoffMs = MAX_BACKOFF_INTERVAL_MS, commitThresholdBytes = COMMIT_THRESHOLD_BYTES) {
 		this.#lc = lc.withContext("component", "backfill-manager");
 		this.#changeStreamer = changeStreamer;
@@ -102,7 +104,7 @@ var BackfillManager = class {
 	#retryDelayMs;
 	#backfillRetryTimer;
 	#checkAndStartBackfill() {
-		if (!this.#backfillRetryTimer && !this.#runningBackfill && this.#requiredBackfills.size) {
+		if (!this.#canceled && !this.#backfillRetryTimer && !this.#runningBackfill && this.#requiredBackfills.size) {
 			const candidates = [...this.#requiredBackfills.values()];
 			const request = candidates[randInt(0, candidates.length - 1)];
 			const state = {
@@ -121,6 +123,10 @@ var BackfillManager = class {
 		}
 	}
 	#retryBackfillWithBackoff(e) {
+		if (this.#canceled) {
+			this.#lc.debug?.(`not retrying backfill: change stream canceled`, e);
+			return;
+		}
 		const log = this.#retryDelayMs === this.#maxBackoffMs ? "error" : "warn";
 		this.#lc[log]?.(`Error running backfill. Retrying in ${this.#retryDelayMs} ms`, e);
 		this.#backfillRetryTimer = setTimeout(() => {
@@ -177,6 +183,10 @@ var BackfillManager = class {
 			uncommittedBytes = 0;
 		};
 		for await (const { message: msg, byteSize } of this.#backfillStreamer(state.request)) {
+			if (this.#canceled) {
+				lc.info?.(`backfill stream canceled: change stream canceled`);
+				return;
+			}
 			const mustWaitBeforeFlush = msg.tag === "backfill-completed" && (this.#changeStreamReached(lc, msg.watermark) || backfillTx !== null && backfillTx < msg.watermark);
 			if (backfillTx && (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush || uncommittedBytes >= this.#commitThresholdBytes)) commitTx();
 			mustWaitBeforeFlush && await mustWaitBeforeFlush;
@@ -387,8 +397,11 @@ var BackfillManager = class {
 		}
 	}
 	cancel() {
+		this.#canceled = true;
 		this.#stopRunningBackfill(`change stream canceled`);
 		clearTimeout(this.#backfillRetryTimer);
+		this.#backfillRetryTimer = void 0;
+		for (const { reached } of this.#awaitingStatusWatermarks.splice(0)) reached();
 	}
 };
 var BackfillStreamError = class extends Error {

package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#commitThresholdBytes","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\n/**\n * A {@link MessageBackfill} or {@link BackfillCompleted} paired with the\n * approximate number of upstream bytes it represents. The BackfillManager uses\n * `byteSize` to bound the size of each replica transaction, committing and\n * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.\n */\nexport type BackfillMessage = {\n  message: MessageBackfill | BackfillCompleted;\n  byteSize: number;\n};\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<BackfillMessage>;\n\n/**\n * Commit (and reopen) the current backfill transaction once this many bytes of\n * backfill data have accumulated in it. Bounds the size of a single replica\n * COMMIT/checkpoint so it can't monopolize the replica.\n */\nconst COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  canceledReason?: string | undefined;\n  minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  readonly #commitThresholdBytes: number;\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n    commitThresholdBytes = COMMIT_THRESHOLD_BYTES,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n    this.#commitThresholdBytes = commitThresholdBytes;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {request, minWatermark: ''};\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    const log = this.#retryDelayMs === this.#maxBackoffMs ? 'error' : 'warn';\n    this.#lc[log]?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n    let uncommittedBytes = 0;\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.canceledReason ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (state.canceledReason === undefined) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      let tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      if (msg.tag === 'backfill-completed' && tx < msg.watermark) {\n        // At this point it must be the case that the #changeStreamReached() the\n        // backfill watermark. Given that guarantee, ensure that the version of the\n        // transaction containing the backfill-completed message is at least up\n        // to the backfill watermark, so that the final database state version is\n        // never earlier than the version of any backfilled rows.\n        tx = msg.watermark;\n      }\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n      }\n      backfillTx = null;\n      uncommittedBytes = 0;\n    };\n\n    for await (const {message: msg, byteSize} of this.#backfillStreamer(\n      state.request,\n    )) {\n      // Before sending `backfill-completed`, the main replication stream\n      // may need to catch up, and/or the current transaction may need to be\n      // committed to open a new transaction that's up to backfill watermark.\n      const mustWaitBeforeFlush =\n        msg.tag === 'backfill-completed' &&\n        (this.#changeStreamReached(lc, msg.watermark) ||\n          (backfillTx !== null && backfillTx < msg.watermark));\n\n      // Commit (and later reopen) the transaction if the main stream is\n      // waiting on the reservation, if we must catch up before completing, or\n      // if the size of current transaction has reached the commit threshold.\n      if (\n        backfillTx &&\n        (changeStream.waiterDelay() > 0 ||\n          mustWaitBeforeFlush ||\n          uncommittedBytes >= this.#commitThresholdBytes)\n      ) {\n        commitTx();\n      }\n\n      mustWaitBeforeFlush && (await mustWaitBeforeFlush);\n\n      if (\n        msg.tag === 'backfill' &&\n        msg.rowValues.length > 0 &&\n        msg.relation.rowKey.columns.length === 0\n      ) {\n        throw new MissingRowKeyError(state.request);\n      }\n\n      // Reserve the changeStreamer if not in a transaction.\n      if ((backfillTx ??= await beginTxFor(msg)) === null) {\n        lc.info?.(\n          `backfill stream canceled: ${state.canceledReason}`,\n          state.request,\n        );\n        this.#checkAndStartBackfill(); // start the next backfill if present\n        return; // this backfill is canceled\n      }\n\n      // `await` to allow the change streamer to exert back pressure\n      // on backfills.\n      await changeStream.push(['data', msg]);\n      uncommittedBytes += byteSize;\n    }\n\n    // Flush any final tx and release the stream.\n    backfillTx && commitTx();\n    lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.canceledReason = reason;\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: backfill,\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: backfill},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: backfill,\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"mappings":";;;;;;;;;AAyBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;;;;;;AAsBA,IAAM,yBAAyB,IAAI,OAAO;AAQ1C,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC;CAEA,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf,uBAAuB,wBACvB;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKE,kBAAkB;EACvB,KAAKC,oBAAoB;EACzB,KAAKC,cAAc;EACnB,KAAKG,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKH,wBAAwB;CAC/B;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAKN,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAKU,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAKC,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAKC,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAKF,wBAAwB,MAAM,WAAW;GACjD,KAAKA,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAKL,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAKA,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAKA,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAKK,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAKL,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAKK,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAKG,uBACN,CAAC,KAAKC,oBACN,KAAKb,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAKA,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IAAC;IAAS,cAAc;GAAE;GACxC,MAAM,KAAK,KAAKD,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAKc,mBAAmB;GACxB,KAAUC,aAAa,IAAI,KAAK,EAC7B,WAAW;IACV,KAAKC,qBAAqB,mBAAmB,KAAK;IAClD,KAAKP,gBAAgB,KAAKF;GAC5B,CAAC,EAGA,OAAM,MAAK;IACV,KAAKS,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAKC,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,MAAM,MAAM,KAAKR,kBAAkB,KAAKD,gBAAgB,UAAU;EAClE,KAAKR,IAAI,OACP,uCAAuC,KAAKS,cAAc,MAC1D,CACF;EACA,KAAKI,sBAAsB,iBAAiB;GAC1C,KAAKA,sBAAsB,KAAA;GAC3B,KAAKD,uBAAuB;EAC9B,GAAG,KAAKH,aAAa;EAErB,KAAKA,gBAAgB,KAAK,IAAI,KAAKA,gBAAgB,GAAG,KAAKD,aAAa;CAC1E;CAEA,MAAMO,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAKb;EAI1B,IAAI,aAA4B;EAChC,IAAI,mBAAmB;;;;EAKvB,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,MAAM,mBAAmB,KAAA,GAAW;KACtC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAKc,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,IAAI,QAAQ,wBAAwB,KAAK,IAAI,WAM/C,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAKZ;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;GACjC;GACA,aAAa;GACb,mBAAmB;EACrB;EAEA,WAAW,MAAM,EAAC,SAAS,KAAK,cAAa,KAAKD,kBAChD,MAAM,OACR,GAAG;GAID,MAAM,sBACJ,IAAI,QAAQ,yBACX,KAAKe,qBAAqB,IAAI,IAAI,SAAS,KACzC,eAAe,QAAQ,aAAa,IAAI;GAK7C,IACE,eACC,aAAa,YAAY,IAAI,KAC5B,uBACA,oBAAoB,KAAKZ,wBAE3B,SAAS;GAGX,uBAAwB,MAAM;GAE9B,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;GAI5C,KAAK,eAAe,MAAM,WAAW,GAAG,OAAO,MAAM;IACnD,GAAG,OACD,6BAA6B,MAAM,kBACnC,MAAM,OACR;IACA,KAAKM,uBAAuB;IAC5B;GACF;GAIA,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;GACrC,oBAAoB;EACtB;EAGA,cAAc,SAAS;EACvB,GAAG,QAAQ,0BAA0B,MAAM,kBAAkB,EAAE;CACjE;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAKE;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAKA;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,iBAAiB;GAC1B,KAAKA,mBAAmB;GACxB,UAAU,KAAKd,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAKC,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAKC,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAKA,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAKC,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAKkB,sBAAsB,QAAQ,GAAG;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKA,sBAAsB;GAC3B,KAAKC,wBAAwB,QAAQ,EAAE;GAGvC,KAAKR,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKQ,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAKnB,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAKU,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAKU,oBAAoB,KAAK,GAChC,KAAKL,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAKL,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS;IACX,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAKV,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAKqB,wBAAwB,KAAK,GAAG;KACrC,KAAKX,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAKU,oBAAoB,GAAG,GAC9B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAKf,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAKqB,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAKD,oBAAoB,EAAE,GAC7B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAKf,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAKU,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,SAAQ;KACnC,CAAC;UAED,KAAKA,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO;MACjB;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAKV,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAKU,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAKU,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAKL,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAKf,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,EAAE,WAAW,GACpC,KAAKqB,wBAAwB,KAAK,KAAK;UAEvC,KAAKX,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAKU,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAKL,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAKK,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAKF,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAKnB,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAKC,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,EAAE,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAKqB,wBAAwB,KAAK,QAAQ;SAE1C,KAAKX,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAKK,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAKA,qBAAqB,wBAAwB;EAClD,aAAa,KAAKH,mBAAmB;CACvC;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}
+{"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#commitThresholdBytes","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#canceled","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\n/**\n * A {@link MessageBackfill} or {@link BackfillCompleted} paired with the\n * approximate number of upstream bytes it represents. The BackfillManager uses\n * `byteSize` to bound the size of each replica transaction, committing and\n * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.\n */\nexport type BackfillMessage = {\n  message: MessageBackfill | BackfillCompleted;\n  byteSize: number;\n};\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<BackfillMessage>;\n\n/**\n * Commit (and reopen) the current backfill transaction once this many bytes of\n * backfill data have accumulated in it. Bounds the size of a single replica\n * COMMIT/checkpoint so it can't monopolize the replica.\n */\nconst COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  canceledReason?: string | undefined;\n  minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  readonly #commitThresholdBytes: number;\n\n  /** Set when the change stream is canceled. No further backfills are run. */\n  #canceled = false;\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n    commitThresholdBytes = COMMIT_THRESHOLD_BYTES,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n    this.#commitThresholdBytes = commitThresholdBytes;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#canceled &&\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {request, minWatermark: ''};\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    if (this.#canceled) {\n      this.#lc.debug?.(`not retrying backfill: change stream canceled`, e);\n      return;\n    }\n    const log = this.#retryDelayMs === this.#maxBackoffMs ? 'error' : 'warn';\n    this.#lc[log]?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n    let uncommittedBytes = 0;\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.canceledReason ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (state.canceledReason === undefined) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      let tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      if (msg.tag === 'backfill-completed' && tx < msg.watermark) {\n        // At this point it must be the case that the #changeStreamReached() the\n        // backfill watermark. Given that guarantee, ensure that the version of the\n        // transaction containing the backfill-completed message is at least up\n        // to the backfill watermark, so that the final database state version is\n        // never earlier than the version of any backfilled rows.\n        tx = msg.watermark;\n      }\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n      }\n      backfillTx = null;\n      uncommittedBytes = 0;\n    };\n\n    for await (const {message: msg, byteSize} of this.#backfillStreamer(\n      state.request,\n    )) {\n      if (this.#canceled) {\n        // Exiting the loop finalizes the backfill stream (and the upstream\n        // resources it holds). The reservation, if held, does not need to be\n        // released since the change stream is gone.\n        lc.info?.(`backfill stream canceled: change stream canceled`);\n        return;\n      }\n      // Before sending `backfill-completed`, the main replication stream\n      // may need to catch up, and/or the current transaction may need to be\n      // committed to open a new transaction that's up to backfill watermark.\n      const mustWaitBeforeFlush =\n        msg.tag === 'backfill-completed' &&\n        (this.#changeStreamReached(lc, msg.watermark) ||\n          (backfillTx !== null && backfillTx < msg.watermark));\n\n      // Commit (and later reopen) the transaction if the main stream is\n      // waiting on the reservation, if we must catch up before completing, or\n      // if the size of current transaction has reached the commit threshold.\n      if (\n        backfillTx &&\n        (changeStream.waiterDelay() > 0 ||\n          mustWaitBeforeFlush ||\n          uncommittedBytes >= this.#commitThresholdBytes)\n      ) {\n        commitTx();\n      }\n\n      mustWaitBeforeFlush && (await mustWaitBeforeFlush);\n\n      if (\n        msg.tag === 'backfill' &&\n        msg.rowValues.length > 0 &&\n        msg.relation.rowKey.columns.length === 0\n      ) {\n        throw new MissingRowKeyError(state.request);\n      }\n\n      // Reserve the changeStreamer if not in a transaction.\n      if ((backfillTx ??= await beginTxFor(msg)) === null) {\n        lc.info?.(\n          `backfill stream canceled: ${state.canceledReason}`,\n          state.request,\n        );\n        this.#checkAndStartBackfill(); // start the next backfill if present\n        return; // this backfill is canceled\n      }\n\n      // `await` to allow the change streamer to exert back pressure\n      // on backfills.\n      await changeStream.push(['data', msg]);\n      uncommittedBytes += byteSize;\n    }\n\n    // Flush any final tx and release the stream.\n    backfillTx && commitTx();\n    lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.canceledReason = reason;\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: backfill,\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: backfill},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: backfill,\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#canceled = true;\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n    this.#backfillRetryTimer = undefined;\n\n    // Wake up a backfill that is waiting for the change stream to reach a\n    // watermark. The change stream never will, and the backfill must be\n    // allowed to unwind (and release its upstream resources) rather than\n    // remain suspended forever.\n    for (const {reached} of this.#awaitingStatusWatermarks.splice(0)) {\n      reached();\n    }\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"mappings":";;;;;;;;;AAyBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;;;;;;AAsBA,IAAM,yBAAyB,IAAI,OAAO;AAQ1C,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC;;CAGA,YAAY;CAEZ,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf,uBAAuB,wBACvB;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKE,kBAAkB;EACvB,KAAKC,oBAAoB;EACzB,KAAKC,cAAc;EACnB,KAAKG,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKH,wBAAwB;CAC/B;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAKN,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAKU,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAKC,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAKC,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAKF,wBAAwB,MAAM,WAAW;GACjD,KAAKA,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAKL,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAKA,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAKA,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAKK,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAKL,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAKK,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAKG,aACN,CAAC,KAAKC,uBACN,CAAC,KAAKC,oBACN,KAAKd,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAKA,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IAAC;IAAS,cAAc;GAAE;GACxC,MAAM,KAAK,KAAKD,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAKe,mBAAmB;GACxB,KAAUC,aAAa,IAAI,KAAK,EAC7B,WAAW;IACV,KAAKC,qBAAqB,mBAAmB,KAAK;IAClD,KAAKR,gBAAgB,KAAKF;GAC5B,CAAC,EAGA,OAAM,MAAK;IACV,KAAKU,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAKC,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,IAAI,KAAKL,WAAW;GAClB,KAAKb,IAAI,QAAQ,iDAAiD,CAAC;GACnE;EACF;EACA,MAAM,MAAM,KAAKS,kBAAkB,KAAKD,gBAAgB,UAAU;EAClE,KAAKR,IAAI,OACP,uCAAuC,KAAKS,cAAc,MAC1D,CACF;EACA,KAAKK,sBAAsB,iBAAiB;GAC1C,KAAKA,sBAAsB,KAAA;GAC3B,KAAKF,uBAAuB;EAC9B,GAAG,KAAKH,aAAa;EAErB,KAAKA,gBAAgB,KAAK,IAAI,KAAKA,gBAAgB,GAAG,KAAKD,aAAa;CAC1E;CAEA,MAAMQ,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAKd;EAI1B,IAAI,aAA4B;EAChC,IAAI,mBAAmB;;;;EAKvB,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,MAAM,mBAAmB,KAAA,GAAW;KACtC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAKe,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,IAAI,QAAQ,wBAAwB,KAAK,IAAI,WAM/C,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAKb;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;GACjC;GACA,aAAa;GACb,mBAAmB;EACrB;EAEA,WAAW,MAAM,EAAC,SAAS,KAAK,cAAa,KAAKD,kBAChD,MAAM,OACR,GAAG;GACD,IAAI,KAAKU,WAAW;IAIlB,GAAG,OAAO,kDAAkD;IAC5D;GACF;GAIA,MAAM,sBACJ,IAAI,QAAQ,yBACX,KAAKM,qBAAqB,IAAI,IAAI,SAAS,KACzC,eAAe,QAAQ,aAAa,IAAI;GAK7C,IACE,eACC,aAAa,YAAY,IAAI,KAC5B,uBACA,oBAAoB,KAAKb,wBAE3B,SAAS;GAGX,uBAAwB,MAAM;GAE9B,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;GAI5C,KAAK,eAAe,MAAM,WAAW,GAAG,OAAO,MAAM;IACnD,GAAG,OACD,6BAA6B,MAAM,kBACnC,MAAM,OACR;IACA,KAAKM,uBAAuB;IAC5B;GACF;GAIA,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;GACrC,oBAAoB;EACtB;EAGA,cAAc,SAAS;EACvB,GAAG,QAAQ,0BAA0B,MAAM,kBAAkB,EAAE;CACjE;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAKG;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAKA;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,iBAAiB;GAC1B,KAAKA,mBAAmB;GACxB,UAAU,KAAKf,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAKC,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAKC,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAKA,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAKC,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAKmB,sBAAsB,QAAQ,GAAG;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKA,sBAAsB;GAC3B,KAAKC,wBAAwB,QAAQ,EAAE;GAGvC,KAAKT,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKS,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAKpB,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAKU,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAKW,oBAAoB,KAAK,GAChC,KAAKL,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAKN,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS;IACX,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAKV,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAKsB,wBAAwB,KAAK,GAAG;KACrC,KAAKZ,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAKW,oBAAoB,GAAG,GAC9B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAKhB,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAKsB,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAKD,oBAAoB,EAAE,GAC7B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAKhB,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAKU,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,SAAQ;KACnC,CAAC;UAED,KAAKA,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO;MACjB;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAKV,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAKU,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAKW,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAKL,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAKhB,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,EAAE,WAAW,GACpC,KAAKsB,wBAAwB,KAAK,KAAK;UAEvC,KAAKZ,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAKW,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAKL,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAKK,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAKF,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAKpB,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAKC,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,EAAE,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAKsB,wBAAwB,KAAK,QAAQ;SAE1C,KAAKZ,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAKM,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAKJ,YAAY;EACjB,KAAKI,qBAAqB,wBAAwB;EAClD,aAAa,KAAKH,mBAAmB;EACrC,KAAKA,sBAAsB,KAAA;EAM3B,KAAK,MAAM,EAAC,aAAY,KAAKT,0BAA0B,OAAO,CAAC,GAC7D,QAAQ;CAEZ;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-stream-multiplexer.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,2BAA2B,CAAC;AAEtD,OAAO,KAAK,EACV,mBAAmB,EACnB,uBAAuB,EACxB,MAAM,wBAAwB,CAAC;AAEhC,MAAM,MAAM,UAAU,GAAG;IACvB,MAAM,IAAI,IAAI,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAAC;CAC7C,CAAC;AAQF;;;GAGG;AACH,qBAAa,uBAAuB;;IAmBlC,YAAY,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,EAMhD;IAED,YAAY,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,GAAG,IAAI,CAGrC;IAED,YAAY,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,GAAG,IAAI,CAGnC;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAelD;IAED;;;;OAIG;IACH,WAAW,IAAI,MAAM,CAKpB;IAED;;;OAGG;IACH,OAAO,CAAC,YAAY,EAAE,MAAM,QAY3B;IAED;;;;OAIG;IACH,UAAU,CAAC,OAAO,EAAE,uBAAuB,QAS1C;IAED;;;OAGG;IACH,IAAI,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,CAOnD;IAED,IAAI,CAAC,GAAG,EAAE,KAAK,QAEd;IAED,QAAQ,IAAI,MAAM,CAAC,mBAAmB,CAAC,CAEtC;CACF"}
+{"version":3,"file":"change-stream-multiplexer.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,2BAA2B,CAAC;AAEtD,OAAO,KAAK,EACV,mBAAmB,EACnB,uBAAuB,EACxB,MAAM,wBAAwB,CAAC;AAEhC,MAAM,MAAM,UAAU,GAAG;IACvB,MAAM,IAAI,IAAI,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAAC;CAC7C,CAAC;AASF;;;GAGG;AACH,qBAAa,uBAAuB;;IAqBlC,YAAY,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,EAMhD;IAmBD,YAAY,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,GAAG,IAAI,CAGrC;IAED,YAAY,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,GAAG,IAAI,CAGnC;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA2BlD;IAED;;;;OAIG;IACH,WAAW,IAAI,MAAM,CAKpB;IAED;;;OAGG;IACH,OAAO,CAAC,YAAY,EAAE,MAAM,QAY3B;IAED;;;;OAIG;IACH,UAAU,CAAC,OAAO,EAAE,uBAAuB,QAS1C;IAED;;;OAGG;IACH,IAAI,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,CAOnD;IAED,IAAI,CAAC,GAAG,EAAE,KAAK,QAEd;IAED,QAAQ,IAAI,MAAM,CAAC,mBAAmB,CAAC,CAEtC;CACF"}

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js

@@ -1,4 +1,5 @@
 import { assert } from "../../../../../shared/src/asserts.js";
+import { AbortError } from "../../../../../shared/src/abort-error.js";
 import { Subscription } from "../../../types/subscription.js";
 import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts
@@ -22,11 +23,20 @@ var ChangeStreamMultiplexer = class {
 	* Tracks the queue of producers waiting for a reservation.
 	*/
 	#waiters = [];
+	#canceled = false;
 	constructor(lc, lastWatermark) {
 		this.#lc = lc;
-		this.#sub = Subscription.create({ cleanup: () => this.#producers.forEach((p) => p.cancel()) });
+		this.#sub = Subscription.create({ cleanup: () => this.#cancel() });
 		this.#lastWatermark = lastWatermark;
 	}
+	#cancel() {
+		this.#canceled = true;
+		this.#producers.forEach((p) => p.cancel());
+		for (const { producer, rejectReservation } of this.#waiters.splice(0)) {
+			this.#lc.info?.(`rejecting reservation request from ${producer}: stream canceled`);
+			rejectReservation(new AbortError("change stream canceled"));
+		}
+	}
 	addProducers(...p) {
 		this.#producers.push(...p);
 		return this;
@@ -46,17 +56,19 @@ var ChangeStreamMultiplexer = class {
 	* @param producer The name of the producer, purely for debugging output
 	*/
 	reserve(producer) {
+		if (this.#canceled) return Promise.reject(new AbortError("change stream canceled"));
 		if (this.#lastWatermark !== null) {
 			const lastWatermark = this.#lastWatermark;
 			this.#lastWatermark = null;
 			return lastWatermark;
 		}
 		const startTime = performance.now();
-		const { promise, resolve: grantReservation } = resolver();
+		const { promise, resolve: grantReservation, reject: rejectReservation } = resolver();
 		this.#waiters.push({
 			producer,
 			startTime,
-			grantReservation
+			grantReservation,
+			rejectReservation
 		});
 		return promise;
 	}

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-stream-multiplexer.js","names":["#lc","#sub","#producers","#listeners","#waiters","#lastWatermark"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Source} from '../../../types/streams.ts';\nimport {Subscription} from '../../../types/subscription.ts';\nimport type {\n  ChangeStreamMessage,\n  DownstreamStatusMessage,\n} from '../protocol/current.ts';\n\nexport type Cancelable = {\n  cancel(): void;\n};\n\nexport type Listener = {\n  onChange(change: ChangeStreamMessage): void;\n};\n\ntype Waiter = {\n  producer: string;\n  startTime: number;\n  grantReservation: (watermark: string) => void;\n};\n\n/**\n * Facilitates cooperative multiplexing of transactions from\n * multiple producers into a single change stream.\n */\nexport class ChangeStreamMultiplexer {\n  readonly #lc: LogContext;\n  readonly #sub: Subscription<ChangeStreamMessage>;\n  readonly #producers: Cancelable[] = [];\n  readonly #listeners: Listener[] = [];\n\n  /**\n   * The `#lastWatermark` tracks the watermark of the last transaction\n   * committed to stream. It is set to null when a producer has reserved\n   * the stream, and set to the new watermark when the producer releases\n   * the reservation.\n   */\n  #lastWatermark: string | null;\n\n  /**\n   * Tracks the queue of producers waiting for a reservation.\n   */\n  readonly #waiters: Waiter[] = [];\n\n  constructor(lc: LogContext, lastWatermark: string) {\n    this.#lc = lc;\n    this.#sub = Subscription.create<ChangeStreamMessage>({\n      cleanup: () => this.#producers.forEach(p => p.cancel()),\n    });\n    this.#lastWatermark = lastWatermark;\n  }\n\n  addProducers(...p: Cancelable[]): this {\n    this.#producers.push(...p);\n    return this;\n  }\n\n  addListeners(...l: Listener[]): this {\n    this.#listeners.push(...l);\n    return this;\n  }\n\n  /**\n   * Called by a producer to \"reserve\" the exclusive right to push data\n   * changes to the stream. If the stream is currently reserved,\n   * the resulting Promise must be awaited.\n   *\n   * The producer must then {@link release()} the reservation when it sees\n   * fit.\n   *\n   * @param producer The name of the producer, purely for debugging output\n   */\n  reserve(producer: string): string | Promise<string> {\n    if (this.#lastWatermark !== null) {\n      // If the stream is not reserved, reserve it and return the\n      // watermark.\n      const lastWatermark = this.#lastWatermark;\n      this.#lastWatermark = null;\n      return lastWatermark;\n    }\n\n    // Otherwise, wait for the current reservation to be released.\n    const startTime = performance.now();\n    const {promise, resolve: grantReservation} = resolver<string>();\n    this.#waiters.push({producer, startTime, grantReservation});\n\n    return promise;\n  }\n\n  /**\n   * If there are producers currently awaiting a reservation, returns\n   * the duration (in milliseconds) of the oldest reservation request.\n   * Returns a negative number if there are no waiters.\n   */\n  waiterDelay(): number {\n    if (this.#waiters.length === 0) {\n      return -1;\n    }\n    return performance.now() - this.#waiters[0].startTime;\n  }\n\n  /**\n   * Called by a producer to release its reservation after committing its\n   * last transaction.\n   */\n  release(newWatermark: string) {\n    const waiter = this.#waiters.shift();\n    if (!waiter) {\n      this.#lastWatermark = newWatermark;\n    } else {\n      const {producer, startTime, grantReservation} = waiter;\n      grantReservation(newWatermark);\n      const elapsed = performance.now() - startTime;\n      this.#lc.info?.(\n        `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`,\n      );\n    }\n  }\n\n  /**\n   * `pushStatus()` can be called without a reservation, as it\n   * does not constitute a data change and can appear anywhere\n   * in the stream.\n   */\n  pushStatus(message: DownstreamStatusMessage) {\n    // Let listeners know about all status messages.\n    this.#listeners.forEach(l => l.onChange(message));\n    // The ChangeStreamer only cares about status messages requiring an ack\n    // or containing a lagReport. To reduce churn, avoid sending other status\n    // messages.\n    if (message[1].ack || message[1].lagReport) {\n      this.#sub.push(message);\n    }\n  }\n\n  /**\n   * `push()` must only be called by a producer after it has\n   * {@link reserve}d the stream.\n   */\n  push(message: ChangeStreamMessage): Promise<unknown> {\n    assert(\n      this.#lastWatermark === null,\n      `push() called without reserve()-ing the stream`,\n    );\n    this.#listeners.forEach(l => l.onChange(message));\n    return this.#sub.push(message).result;\n  }\n\n  fail(err: Error) {\n    this.#sub.fail(err);\n  }\n\n  asSource(): Source<ChangeStreamMessage> {\n    return this.#sub;\n  }\n}\n"],"mappings":";;;;;;;;AA4BA,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA,aAAoC,CAAC;CACrC,aAAkC,CAAC;;;;;;;CAQnC;;;;CAKA,WAA8B,CAAC;CAE/B,YAAY,IAAgB,eAAuB;EACjD,KAAKA,MAAM;EACX,KAAKC,OAAO,aAAa,OAA4B,EACnD,eAAe,KAAKC,WAAW,SAAQ,MAAK,EAAE,OAAO,CAAC,EACxD,CAAC;EACD,KAAKG,iBAAiB;CACxB;CAEA,aAAa,GAAG,GAAuB;EACrC,KAAKH,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;CAEA,aAAa,GAAG,GAAqB;EACnC,KAAKC,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;;;;;;;;;;;CAYA,QAAQ,UAA4C;EAClD,IAAI,KAAKE,mBAAmB,MAAM;GAGhC,MAAM,gBAAgB,KAAKA;GAC3B,KAAKA,iBAAiB;GACtB,OAAO;EACT;EAGA,MAAM,YAAY,YAAY,IAAI;EAClC,MAAM,EAAC,SAAS,SAAS,qBAAoB,SAAiB;EAC9D,KAAKD,SAAS,KAAK;GAAC;GAAU;GAAW;EAAgB,CAAC;EAE1D,OAAO;CACT;;;;;;CAOA,cAAsB;EACpB,IAAI,KAAKA,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO,YAAY,IAAI,IAAI,KAAKA,SAAS,GAAG;CAC9C;;;;;CAMA,QAAQ,cAAsB;EAC5B,MAAM,SAAS,KAAKA,SAAS,MAAM;EACnC,IAAI,CAAC,QACH,KAAKC,iBAAiB;OACjB;GACL,MAAM,EAAC,UAAU,WAAW,qBAAoB;GAChD,iBAAiB,YAAY;GAC7B,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,KAAKL,IAAI,OACP,GAAG,SAAS,UAAU,QAAQ,QAAQ,CAAC,EAAE,2BAC3C;EACF;CACF;;;;;;CAOA,WAAW,SAAkC;EAE3C,KAAKG,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAIhD,IAAI,QAAQ,GAAG,OAAO,QAAQ,GAAG,WAC/B,KAAKF,KAAK,KAAK,OAAO;CAE1B;;;;;CAMA,KAAK,SAAgD;EACnD,OACE,KAAKI,mBAAmB,MACxB,gDACF;EACA,KAAKF,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAChD,OAAO,KAAKF,KAAK,KAAK,OAAO,EAAE;CACjC;CAEA,KAAK,KAAY;EACf,KAAKA,KAAK,KAAK,GAAG;CACpB;CAEA,WAAwC;EACtC,OAAO,KAAKA;CACd;AACF"}
+{"version":3,"file":"change-stream-multiplexer.js","names":["#lc","#sub","#producers","#listeners","#waiters","#cancel","#lastWatermark","#canceled"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Source} from '../../../types/streams.ts';\nimport {Subscription} from '../../../types/subscription.ts';\nimport type {\n  ChangeStreamMessage,\n  DownstreamStatusMessage,\n} from '../protocol/current.ts';\n\nexport type Cancelable = {\n  cancel(): void;\n};\n\nexport type Listener = {\n  onChange(change: ChangeStreamMessage): void;\n};\n\ntype Waiter = {\n  producer: string;\n  startTime: number;\n  grantReservation: (watermark: string) => void;\n  rejectReservation: (err: Error) => void;\n};\n\n/**\n * Facilitates cooperative multiplexing of transactions from\n * multiple producers into a single change stream.\n */\nexport class ChangeStreamMultiplexer {\n  readonly #lc: LogContext;\n  readonly #sub: Subscription<ChangeStreamMessage>;\n  readonly #producers: Cancelable[] = [];\n  readonly #listeners: Listener[] = [];\n\n  /**\n   * The `#lastWatermark` tracks the watermark of the last transaction\n   * committed to stream. It is set to null when a producer has reserved\n   * the stream, and set to the new watermark when the producer releases\n   * the reservation.\n   */\n  #lastWatermark: string | null;\n\n  /**\n   * Tracks the queue of producers waiting for a reservation.\n   */\n  readonly #waiters: Waiter[] = [];\n\n  #canceled = false;\n\n  constructor(lc: LogContext, lastWatermark: string) {\n    this.#lc = lc;\n    this.#sub = Subscription.create<ChangeStreamMessage>({\n      cleanup: () => this.#cancel(),\n    });\n    this.#lastWatermark = lastWatermark;\n  }\n\n  #cancel() {\n    this.#canceled = true;\n    this.#producers.forEach(p => p.cancel());\n\n    // A producer awaiting a reservation would otherwise remain suspended\n    // forever, since the reservation held when the stream was canceled is\n    // never released. Reject the pending reservations so that the producers\n    // unwind and release the resources (e.g. upstream connections and\n    // transactions) that they were holding for the stream.\n    for (const {producer, rejectReservation} of this.#waiters.splice(0)) {\n      this.#lc.info?.(\n        `rejecting reservation request from ${producer}: stream canceled`,\n      );\n      rejectReservation(new AbortError('change stream canceled'));\n    }\n  }\n\n  addProducers(...p: Cancelable[]): this {\n    this.#producers.push(...p);\n    return this;\n  }\n\n  addListeners(...l: Listener[]): this {\n    this.#listeners.push(...l);\n    return this;\n  }\n\n  /**\n   * Called by a producer to \"reserve\" the exclusive right to push data\n   * changes to the stream. If the stream is currently reserved,\n   * the resulting Promise must be awaited.\n   *\n   * The producer must then {@link release()} the reservation when it sees\n   * fit.\n   *\n   * @param producer The name of the producer, purely for debugging output\n   */\n  reserve(producer: string): string | Promise<string> {\n    if (this.#canceled) {\n      return Promise.reject(new AbortError('change stream canceled'));\n    }\n    if (this.#lastWatermark !== null) {\n      // If the stream is not reserved, reserve it and return the\n      // watermark.\n      const lastWatermark = this.#lastWatermark;\n      this.#lastWatermark = null;\n      return lastWatermark;\n    }\n\n    // Otherwise, wait for the current reservation to be released.\n    const startTime = performance.now();\n    const {\n      promise,\n      resolve: grantReservation,\n      reject: rejectReservation,\n    } = resolver<string>();\n    this.#waiters.push({\n      producer,\n      startTime,\n      grantReservation,\n      rejectReservation,\n    });\n\n    return promise;\n  }\n\n  /**\n   * If there are producers currently awaiting a reservation, returns\n   * the duration (in milliseconds) of the oldest reservation request.\n   * Returns a negative number if there are no waiters.\n   */\n  waiterDelay(): number {\n    if (this.#waiters.length === 0) {\n      return -1;\n    }\n    return performance.now() - this.#waiters[0].startTime;\n  }\n\n  /**\n   * Called by a producer to release its reservation after committing its\n   * last transaction.\n   */\n  release(newWatermark: string) {\n    const waiter = this.#waiters.shift();\n    if (!waiter) {\n      this.#lastWatermark = newWatermark;\n    } else {\n      const {producer, startTime, grantReservation} = waiter;\n      grantReservation(newWatermark);\n      const elapsed = performance.now() - startTime;\n      this.#lc.info?.(\n        `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`,\n      );\n    }\n  }\n\n  /**\n   * `pushStatus()` can be called without a reservation, as it\n   * does not constitute a data change and can appear anywhere\n   * in the stream.\n   */\n  pushStatus(message: DownstreamStatusMessage) {\n    // Let listeners know about all status messages.\n    this.#listeners.forEach(l => l.onChange(message));\n    // The ChangeStreamer only cares about status messages requiring an ack\n    // or containing a lagReport. To reduce churn, avoid sending other status\n    // messages.\n    if (message[1].ack || message[1].lagReport) {\n      this.#sub.push(message);\n    }\n  }\n\n  /**\n   * `push()` must only be called by a producer after it has\n   * {@link reserve}d the stream.\n   */\n  push(message: ChangeStreamMessage): Promise<unknown> {\n    assert(\n      this.#lastWatermark === null,\n      `push() called without reserve()-ing the stream`,\n    );\n    this.#listeners.forEach(l => l.onChange(message));\n    return this.#sub.push(message).result;\n  }\n\n  fail(err: Error) {\n    this.#sub.fail(err);\n  }\n\n  asSource(): Source<ChangeStreamMessage> {\n    return this.#sub;\n  }\n}\n"],"mappings":";;;;;;;;;AA8BA,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA,aAAoC,CAAC;CACrC,aAAkC,CAAC;;;;;;;CAQnC;;;;CAKA,WAA8B,CAAC;CAE/B,YAAY;CAEZ,YAAY,IAAgB,eAAuB;EACjD,KAAKA,MAAM;EACX,KAAKC,OAAO,aAAa,OAA4B,EACnD,eAAe,KAAKI,QAAQ,EAC9B,CAAC;EACD,KAAKC,iBAAiB;CACxB;CAEA,UAAU;EACR,KAAKC,YAAY;EACjB,KAAKL,WAAW,SAAQ,MAAK,EAAE,OAAO,CAAC;EAOvC,KAAK,MAAM,EAAC,UAAU,uBAAsB,KAAKE,SAAS,OAAO,CAAC,GAAG;GACnE,KAAKJ,IAAI,OACP,sCAAsC,SAAS,kBACjD;GACA,kBAAkB,IAAI,WAAW,wBAAwB,CAAC;EAC5D;CACF;CAEA,aAAa,GAAG,GAAuB;EACrC,KAAKE,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;CAEA,aAAa,GAAG,GAAqB;EACnC,KAAKC,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;;;;;;;;;;;CAYA,QAAQ,UAA4C;EAClD,IAAI,KAAKI,WACP,OAAO,QAAQ,OAAO,IAAI,WAAW,wBAAwB,CAAC;EAEhE,IAAI,KAAKD,mBAAmB,MAAM;GAGhC,MAAM,gBAAgB,KAAKA;GAC3B,KAAKA,iBAAiB;GACtB,OAAO;EACT;EAGA,MAAM,YAAY,YAAY,IAAI;EAClC,MAAM,EACJ,SACA,SAAS,kBACT,QAAQ,sBACN,SAAiB;EACrB,KAAKF,SAAS,KAAK;GACjB;GACA;GACA;GACA;EACF,CAAC;EAED,OAAO;CACT;;;;;;CAOA,cAAsB;EACpB,IAAI,KAAKA,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO,YAAY,IAAI,IAAI,KAAKA,SAAS,GAAG;CAC9C;;;;;CAMA,QAAQ,cAAsB;EAC5B,MAAM,SAAS,KAAKA,SAAS,MAAM;EACnC,IAAI,CAAC,QACH,KAAKE,iBAAiB;OACjB;GACL,MAAM,EAAC,UAAU,WAAW,qBAAoB;GAChD,iBAAiB,YAAY;GAC7B,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,KAAKN,IAAI,OACP,GAAG,SAAS,UAAU,QAAQ,QAAQ,CAAC,EAAE,2BAC3C;EACF;CACF;;;;;;CAOA,WAAW,SAAkC;EAE3C,KAAKG,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAIhD,IAAI,QAAQ,GAAG,OAAO,QAAQ,GAAG,WAC/B,KAAKF,KAAK,KAAK,OAAO;CAE1B;;;;;CAMA,KAAK,SAAgD;EACnD,OACE,KAAKK,mBAAmB,MACxB,gDACF;EACA,KAAKH,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAChD,OAAO,KAAKF,KAAK,KAAK,OAAO,EAAE;CACjC;CAEA,KAAK,KAAY;EACf,KAAKA,KAAK,KAAK,GAAG;CACpB;CAEA,WAAwC;EACtC,OAAO,KAAKA;CACd;AACF"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,aAAc,YAAW,gBAAgB;;IACpD,QAAQ,CAAC,EAAE,oBAAoB;IAS/B,YACE,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,EACrC,cAAc,EAAE,qBAAqB,EACrC,WAAW,EAAE,MAAM,EAMpB;IAED,mBAAmB,kBAElB;IAEK,GAAG,kBAmBR;IAED,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAGpB;CA+BF"}
+{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,aAAc,YAAW,gBAAgB;;IACpD,QAAQ,CAAC,EAAE,oBAAoB;IAiB/B,YACE,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,EACrC,cAAc,EAAE,qBAAqB,EACrC,WAAW,EAAE,MAAM,EAMpB;IAED,mBAAmB,kBAElB;IAEK,GAAG,kBAsBR;IAqDD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAGpB;CA+BF"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js

@@ -11,6 +11,14 @@ var BackupMonitor = class {
 	#replicaFile;
 	#firstBackupReceived = resolver();
 	#latestBackup;
+	#firstBackupResolved = false;
+	/**
+	* The replica's `stateVersion` when the monitor started, i.e. the point the
+	* backup has to reach before it covers what this task is about to serve.
+	* `undefined` when the replica could not be read, which degrades the gate to
+	* "any watermark" rather than hanging startup forever.
+	*/
+	#coverageTarget;
 	constructor(lc, watermarks, changeStreamer, replicaFile) {
 		this.#lc = lc;
 		this.#watermarks = watermarks;
@@ -23,21 +31,64 @@ var BackupMonitor = class {
 	async run() {
 		this.#lc.info?.("starting backup monitor");
 		this.#initBackupLagMetric();
+		this.#coverageTarget = this.#readReplicaStateVersion();
 		for await (const backedUp of this.#watermarks) {
-			if (!this.#latestBackup) this.#firstBackupReceived.resolve();
-			else if (backedUp.watermark < this.#latestBackup.watermark) {
-				this.#lc.warn?.(`ignoring earlier backup watermark`, { backedUp });
-				continue;
-			} else if (backedUp.watermark === this.#latestBackup.watermark) {
-				this.#lc.debug?.(`ignoring redundant backup watermark`, { backedUp });
-				continue;
+			if (this.#latestBackup) {
+				if (backedUp.watermark < this.#latestBackup.watermark) {
+					this.#lc.warn?.(`ignoring earlier backup watermark`, { backedUp });
+					continue;
+				}
+				if (backedUp.watermark === this.#latestBackup.watermark) {
+					this.#lc.debug?.(`ignoring redundant backup watermark`, { backedUp });
+					continue;
+				}
 			}
 			this.#lc.info?.(`received backup watermark`, { backedUp });
 			this.#latestBackup = backedUp;
 			this.#changeStreamer.trackBackupWatermark(backedUp.watermark);
+			this.#checkFirstBackupCovers(backedUp);
 		}
 		this.#lc.info?.("watermark stream closed. BackupMonitor stopped.");
 	}
+	/**
+	* Resolves {@link firstBackupReceived} once the backup actually covers the
+	* replica, rather than on the first watermark of any value.
+	*
+	* A new backup destination starts empty and is filled by re-uploading the
+	* local LTX chain, so the first watermarks read back out of it are real but
+	* far behind the replica. Releasing the readiness gate on one of those lets
+	* the task serve against a backup that does not yet cover it, which is what
+	* demotes a restoring view-syncer to Postgres catchup.
+	*
+	* The target is pinned at startup rather than compared against the replica's
+	* current position, which keeps the gate satisfiable under sustained writes.
+	*/
+	#checkFirstBackupCovers(backedUp) {
+		if (this.#firstBackupResolved) return;
+		const target = this.#coverageTarget;
+		if (target !== void 0 && backedUp.watermark < target) {
+			this.#lc.info?.(`backup does not yet cover the replica`, {
+				backedUp,
+				coverageTarget: target
+			});
+			return;
+		}
+		this.#firstBackupResolved = true;
+		this.#firstBackupReceived.resolve();
+	}
+	#readReplicaStateVersion() {
+		let db;
+		try {
+			db = new Database(this.#lc, this.#replicaFile, { readonly: true });
+			const { stateVersion } = db.prepare(`SELECT stateVersion FROM "_zero.replicationState"`).get();
+			return stateVersion;
+		} catch (e) {
+			this.#lc.warn?.("unable to read the replica's stateVersion; the initial backup gate will accept the first watermark", e);
+			return;
+		} finally {
+			db?.close();
+		}
+	}
 	stop() {
 		this.#watermarks.cancel();
 		return promiseVoid;

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#latestBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debuggin; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n\n    for await (const backedUp of this.#watermarks) {\n      if (!this.#latestBackup) {\n        this.#firstBackupReceived.resolve();\n      } else if (backedUp.watermark < this.#latestBackup.watermark) {\n        this.#lc.warn?.(`ignoring earlier backup watermark`, {backedUp});\n        continue;\n      } else if (backedUp.watermark === this.#latestBackup.watermark) {\n        this.#lc.debug?.(`ignoring redundant backup watermark`, {backedUp});\n        continue;\n      }\n      this.#lc.info?.(`received backup watermark`, {backedUp});\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAKC,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAKJ,IAAI,OAAO,yBAAyB;EACzC,KAAKK,qBAAqB;EAE1B,WAAW,MAAM,YAAY,KAAKJ,aAAa;GAC7C,IAAI,CAAC,KAAKK,eACR,KAAKF,qBAAqB,QAAQ;QAC7B,IAAI,SAAS,YAAY,KAAKE,cAAc,WAAW;IAC5D,KAAKN,IAAI,OAAO,qCAAqC,EAAC,SAAQ,CAAC;IAC/D;GACF,OAAO,IAAI,SAAS,cAAc,KAAKM,cAAc,WAAW;IAC9D,KAAKN,IAAI,QAAQ,uCAAuC,EAAC,SAAQ,CAAC;IAClE;GACF;GACA,KAAKA,IAAI,OAAO,6BAA6B,EAAC,SAAQ,CAAC;GACvD,KAAKM,gBAAgB;GACrB,KAAKJ,gBAAgB,qBAAqB,SAAS,SAAS;EAC9D;EACA,KAAKF,IAAI,OAAO,iDAAiD;CACnE;CAEA,OAAsB;EACpB,KAAKC,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,eAAe,KAAKK;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAKN,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKH,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}
+{"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#coverageTarget","#readReplicaStateVersion","#latestBackup","#checkFirstBackupCovers","#firstBackupResolved"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debuggin; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n  #firstBackupResolved = false;\n  /**\n   * The replica's `stateVersion` when the monitor started, i.e. the point the\n   * backup has to reach before it covers what this task is about to serve.\n   * `undefined` when the replica could not be read, which degrades the gate to\n   * \"any watermark\" rather than hanging startup forever.\n   */\n  #coverageTarget: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n    this.#coverageTarget = this.#readReplicaStateVersion();\n\n    for await (const backedUp of this.#watermarks) {\n      if (this.#latestBackup) {\n        if (backedUp.watermark < this.#latestBackup.watermark) {\n          this.#lc.warn?.(`ignoring earlier backup watermark`, {backedUp});\n          continue;\n        }\n        if (backedUp.watermark === this.#latestBackup.watermark) {\n          this.#lc.debug?.(`ignoring redundant backup watermark`, {backedUp});\n          continue;\n        }\n      }\n      this.#lc.info?.(`received backup watermark`, {backedUp});\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n      this.#checkFirstBackupCovers(backedUp);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  /**\n   * Resolves {@link firstBackupReceived} once the backup actually covers the\n   * replica, rather than on the first watermark of any value.\n   *\n   * A new backup destination starts empty and is filled by re-uploading the\n   * local LTX chain, so the first watermarks read back out of it are real but\n   * far behind the replica. Releasing the readiness gate on one of those lets\n   * the task serve against a backup that does not yet cover it, which is what\n   * demotes a restoring view-syncer to Postgres catchup.\n   *\n   * The target is pinned at startup rather than compared against the replica's\n   * current position, which keeps the gate satisfiable under sustained writes.\n   */\n  #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n    if (this.#firstBackupResolved) {\n      return;\n    }\n    const target = this.#coverageTarget;\n    if (target !== undefined && backedUp.watermark < target) {\n      this.#lc.info?.(`backup does not yet cover the replica`, {\n        backedUp,\n        coverageTarget: target,\n      });\n      return;\n    }\n    this.#firstBackupResolved = true;\n    this.#firstBackupReceived.resolve();\n  }\n\n  #readReplicaStateVersion(): string | undefined {\n    let db;\n    try {\n      db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      const {stateVersion} = db\n        .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n        .get<{stateVersion: string}>();\n      return stateVersion;\n    } catch (e) {\n      // Without a target the gate degrades to its previous behavior. That is\n      // strictly better than blocking startup on a replica we cannot read.\n      this.#lc.warn?.(\n        `unable to read the replica's stateVersion; ` +\n          `the initial backup gate will accept the first watermark`,\n        e,\n      );\n      return undefined;\n    } finally {\n      db?.close();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAKC,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAKJ,IAAI,OAAO,yBAAyB;EACzC,KAAKK,qBAAqB;EAC1B,KAAKC,kBAAkB,KAAKC,yBAAyB;EAErD,WAAW,MAAM,YAAY,KAAKN,aAAa;GAC7C,IAAI,KAAKO,eAAe;IACtB,IAAI,SAAS,YAAY,KAAKA,cAAc,WAAW;KACrD,KAAKR,IAAI,OAAO,qCAAqC,EAAC,SAAQ,CAAC;KAC/D;IACF;IACA,IAAI,SAAS,cAAc,KAAKQ,cAAc,WAAW;KACvD,KAAKR,IAAI,QAAQ,uCAAuC,EAAC,SAAQ,CAAC;KAClE;IACF;GACF;GACA,KAAKA,IAAI,OAAO,6BAA6B,EAAC,SAAQ,CAAC;GACvD,KAAKQ,gBAAgB;GACrB,KAAKN,gBAAgB,qBAAqB,SAAS,SAAS;GAC5D,KAAKO,wBAAwB,QAAQ;EACvC;EACA,KAAKT,IAAI,OAAO,iDAAiD;CACnE;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAKU,sBACP;EAEF,MAAM,SAAS,KAAKJ;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAKN,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAKU,uBAAuB;EAC5B,KAAKN,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAKJ,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,EACnE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAKH,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAKC,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,eAAe,KAAKO;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAKR,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKH,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/snapshot.d.ts

@@ -14,6 +14,7 @@
 import type { LogContext } from '@rocicorp/logger';
 import * as v from '../../../../shared/src/valita.ts';
 import { type NormalizedZeroConfig } from '../../config/normalize.ts';
+import type { Source } from '../../types/streams.ts';
 declare const statusSchema: v.ObjectType<{
     tag: v.Type<"status">;
     /**
@@ -79,6 +80,7 @@ export declare const snapshotMessageSchema: v.UnionType<[v.TupleType<[v.Type<"st
     minWatermark: v.Type<string>;
 }, undefined>]>]>;
 export type SnapshotMessage = v.Infer<typeof statusMessageSchema>;
-export declare function reserveAndGetSnapshotStatus(lc: LogContext, config: NormalizedZeroConfig): Promise<SnapshotStatus>;
+export type ReserveSnapshot = (lc: LogContext, config: NormalizedZeroConfig) => Promise<Source<SnapshotMessage>>;
+export declare function reserveAndGetSnapshotStatus(lc: LogContext, config: NormalizedZeroConfig, reserve?: ReserveSnapshot): Promise<SnapshotStatus>;
 export {};
 //# sourceMappingURL=snapshot.d.ts.map

package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,EAAC,KAAK,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAKpE,QAAA,MAAM,YAAY;;IAGhB;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;aAEH,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAE1D,QAAA,MAAM,mBAAmB;;IAxBvB;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;eAMmE,CAAC;AAEzE,eAAO,MAAM,qBAAqB;;IA1BhC;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;iBAQ4D,CAAC;AAElE,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAElE,wBAAgB,2BAA2B,CACzC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAiDzB"}
+{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,EAAC,KAAK,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,QAAA,MAAM,YAAY;;IAGhB;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;aAEH,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAE1D,QAAA,MAAM,mBAAmB;;IAxBvB;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;eAMmE,CAAC;AAEzE,eAAO,MAAM,qBAAqB;;IA1BhC;;OAEG;;IAGH;;;;;OAKG;;IAGH;;;;;OAKG;;iBAQ4D,CAAC;AAElE,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAElE,MAAM,MAAM,eAAe,GAAG,CAC5B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,KACzB,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;AAEtC,wBAAgB,2BAA2B,CACzC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,OAAO,GAAE,eAAiC,GACzC,OAAO,CAAC,cAAc,CAAC,CAwEzB"}