@rocicorp/zero 1.12.0-canary.8 → 1.12.0-canary.9

package/out/zero-cache/src/server/change-streamer.js

@@ -1,5 +1,6 @@
 import { assert } from "../../../shared/src/asserts.js";
 import { promiseVoid } from "../../../shared/src/resolved-promises.js";
+import { resolver } from "../../../shared/src/resolver.js";
 import { must } from "../../../shared/src/must.js";
 import { broadcastWorker, childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
 import { getShardConfig } from "../types/shards.js";
@@ -33,7 +34,6 @@ import { initializeStreamer } from "../services/change-streamer/change-streamer-
 import { initChangeStreamerSchema } from "../services/change-streamer/schema/init.js";
 import { ServerStartMetrics } from "../services/change-streamer/server-start-metrics.js";
 import { startOtelAuto } from "./otel-start.js";
-import { resolver } from "@rocicorp/resolver";
 import { LogContext, consoleLogSink } from "@rocicorp/logger";
 //#region ../zero-cache/src/server/change-streamer.ts
 var lc = new LogContext("info", {}, consoleLogSink);

package/out/zero-cache/src/server/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {InitCleanup} from '../services/change-source/common/init-cleanup.ts';\nimport type {PgChangeLogPurgeLocker} from '../services/change-source/common/replica-restore.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport type {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {DEFAULT_READINESS_GRACE_PERIOD_MS} from '../services/change-streamer/restore-admission.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {ServerStartMetrics} from '../services/change-streamer/server-start-metrics.ts';\nimport {\n  PurgeLocker,\n  type PurgeLock,\n} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {deleteLitestreamMetaDir} from '../services/litestream/commands.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {warnIfPostgresUnpatched} from '../types/pg-patch.ts';\nimport {connectPgClient, type PostgresDB} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig, type ShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\n// Errors thrown once the services are running are reported by the services\n// themselves (e.g. as 'Replicating' status events), and are thus not\n// published as 'Initializing' errors.\nlet running = false;\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      maxConcurrentSwitchoverRestores,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n  if (upstream.type === 'pg') {\n    warnIfPostgresUnpatched(lc);\n  }\n\n  const shard = getShardConfig(config);\n\n  // The change DB is only used by the PG change log. When it is disabled\n  // (RMv2), no connection to the change DB is made.\n  const changeDB = pgChangeLogEnabled\n    ? await connectChangeDB(lc, change.db, shard)\n    : null;\n\n  let purgeLock = null as PurgeLock | null;\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupMonitor: BackupMonitor | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  const startMetrics = new ServerStartMetrics();\n  let waitForFirstBackupBeforeServing = false;\n  let newBackupLineage = false;\n  for (const first of [true, false]) {\n    // Resources acquired by an initialization attempt (e.g. a claimed\n    // replication slot, the purge lock) to be released if the attempt fails.\n    const cleanup = new InitCleanup(lc);\n    // When restoring from litestream, a lock is acquired to prevent change-log\n    // purges. This ensures that (this) change-streamer will be able to resume\n    // from the backup. The lock is acquired by the change source initialization\n    // just before restoring (i.e. after a replication slot is created, if\n    // applicable), and released once this change-streamer takes over the\n    // change-log.\n    const acquirePurgeLock: PgChangeLogPurgeLocker | undefined =\n      changeDB && litestream.backupURL && litestream.executable\n        ? async (slotWatermark?: string) => {\n            const lock = await new PurgeLocker(lc, shard, changeDB).acquire(\n              slotWatermark,\n            );\n            purgeLock = lock === 'behind-slot' ? null : lock;\n            if (purgeLock) {\n              const acquired = purgeLock;\n              cleanup.onFailure('purge lock', () => {\n                purgeLock = null;\n                return acquired.release();\n              });\n            }\n            return lock;\n          }\n        : undefined;\n\n    const restoreOptions = {litestream, acquirePurgeLock, cleanup};\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        pgReplicationEpoch: epoch,\n        pgReplicationSlotPerReplica: slotPerReplica,\n        pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n      } = upstream;\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n        newBackupLineage: initNewBackupLineage,\n        pgChangeLogBehindSlot,\n        startTimings,\n      } = upstream.type === 'pg'\n        ? await initializePostgresChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            {\n              ...initialSync,\n              replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n            },\n            context,\n            replicationLag.reportIntervalMs,\n            restoreOptions,\n            {\n              epoch,\n              slotPerReplica,\n              inactiveReplicaGracePeriodMs,\n              backupV5: litestream.backupUsingV5,\n              pgChangeLog: pgChangeLogEnabled,\n            },\n            upstream.pgStreamInboundTimeoutMs,\n          )\n        : await initializeCustomChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            context,\n            restoreOptions,\n          );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      // Created before the change-streamer, which tracks its watermarks (and\n      // requests prompt backups from it) from construction. It is run (and\n      // stopped) as a service below. It monitors the backup-replicator's\n      // destination (see `backupURLOverride` below).\n      backupMonitor = createBackupCleanupMonitor({\n        lc,\n        config: destinationBackupURL\n          ? {\n              ...config,\n              litestream: {\n                ...config.litestream,\n                backupURLOverride: destinationBackupURL,\n              },\n            }\n          : config,\n        replicaFile: replica.file,\n      });\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n              replicaFile: replica.file,\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogBehindSlot: pgChangeLogBehindSlot ?? false,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          maxConcurrentSwitchoverRestores,\n          // When lag reporting is enabled, a restored view-syncer waits for\n          // the next lag report (sent at an interval with ~1 second of\n          // jitter) before becoming ready.\n          switchoverRestoreReadinessGracePeriodMs:\n            (replicationLag.reportIntervalMs > 0\n              ? replicationLag.reportIntervalMs + 1000\n              : 0) + DEFAULT_READINESS_GRACE_PERIOD_MS,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n          backupMonitor,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      startMetrics.setKind(startTimings?.kind);\n      startMetrics.record('slot_create', startTimings?.slotCreateMs);\n      startMetrics.record('fork_wait', startTimings?.forkWaitMs);\n      startMetrics.record('restore', startTimings?.restoreMs);\n      newBackupLineage = initNewBackupLineage;\n      break;\n    } catch (e) {\n      // Release the resources acquired by the failed attempt, e.g.:\n      // * The claimed replication slot, which would otherwise remain active\n      //   indefinitely, preventing a retry from resuming (or forking) it.\n      // * The purge lock. This is safe because the purge lock exists to\n      //   preserve change-log entries so the new change-streamer can resume\n      //   from the backup replica's watermark. An AutoResetSignal means we\n      //   can't resume from the backup replica (e.g. its replication slot is\n      //   gone), so the change-log entries the lock was protecting are no\n      //   longer needed. The retry performs a fresh initial sync with a new\n      //   replication slot, independent of the old change-log. Releasing is\n      //   also necessary to avoid a self-deadlock when CHANGE_DB ==\n      //   UPSTREAM_DB: CREATE_REPLICATION_SLOT waits for all older\n      //   transactions to finish, including this lock's open transaction.\n      await cleanup.release();\n      // Not yet started (the change-streamer starts it), so nothing to wait on.\n      void backupMonitor?.stop();\n      backupMonitor = undefined;\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n  assert(backupMonitor, `backup monitor is created with the change-streamer`);\n\n  // The backup-replicator is stopped with the other services, i.e. after the\n  // ChangeStreamerHttpServer has drained its subscribers.\n  const processes = new SupportingProcesses(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  const {promise: replicatorReady, resolve} = resolver();\n  if (backupURL) {\n    const catchUpStart = performance.now();\n    void replicatorReady.then(() =>\n      startMetrics.record('catch_up', performance.now() - catchUpStart),\n    );\n  }\n  if (!backupURL) {\n    resolve(); // No backup replicator to wait for, so resolve immediately.\n  } else {\n    lc.info?.('setting up backup to', backupURL);\n    config.litestream.backupURLOverride = backupURL;\n    if (newBackupLineage) {\n      deleteLitestreamMetaDir(replica.file);\n    }\n    // Start a backup replicator, which starts up the corresponding\n    // litestream backup process.\n    const backupReplicator = processes\n      .add(\n        childWorker(\n          REPLICATOR_URL,\n          {...env, ZERO_LITESTREAM_BACKUP_URL_OVERRIDE: backupURL},\n          'backup' satisfies ReplicaFileMode,\n        ),\n        'backup-replicator',\n      )\n      .onceMessageType('ready', () => resolve());\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n  }\n\n  let backupReady = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    backupReady = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n      startMetrics.record('backup_confirm', elapsed);\n    });\n  }\n\n  // In RMv2, readiness additionally waits for the backup-replicator to catch\n  // up to the replication stream. This is not done in RMv1 (i.e. when the\n  // PG change-log is enabled), in which the change-streamer itself is only\n  // started after the readinessGate (plus a startup delay), and thus the\n  // backup-replicator cannot catch up before then.\n  const readinessGate = pgChangeLogEnabled\n    ? backupReady\n    : Promise.all([backupReady, replicatorReady]);\n\n  // Create the broadcast facade once: each broadcastWorker() adds permanent\n  // 'message' forwarders to every sub worker, so creating one per /profz\n  // request would leak a forwarder per request.\n  const profileWorker =\n    profileSubWorkers.length > 0\n      ? broadcastWorker(profileSubWorkers)\n      : undefined;\n  const getProfileWorker = profileWorker\n    ? () => Promise.resolve(profileWorker)\n    : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      // The startup delay is only relevant when taking over the PG change-log\n      // (RMv1 and RMv1.5), and is disabled for RMv2.\n      startupDelayMs: pgChangeLogEnabled ? startupDelayMs : 0,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => {\n    startMetrics.ready();\n    parent.send(['ready', {ready: true}]);\n  });\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  running = true;\n  await runUntilKilled(\n    lc,\n    parent,\n    changeStreamerWebServer,\n    backupMonitor,\n    processes,\n  );\n}\n\n/**\n * Connects the startup-time client used for change-streamer initialization\n * and handoff / takeover, and ensures that the change DB schema is\n * initialized / up-to-date. Steady-state clients are managed by the\n * change-streamer (Storer) implementation.\n */\nasync function connectChangeDB(\n  lc: LogContext,\n  connectionURI: string,\n  shard: ShardConfig,\n): Promise<PostgresDB> {\n  const changeDB = await connectPgClient(\n    lc,\n    connectionURI,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  return changeDB;\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        if (!running) {\n          await publishCriticalEvent(\n            lc,\n            replicationStatusError(lc, 'Initializing', e),\n          );\n        }\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAKlD,IAAI,UAAU;AAEd,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,iCACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CACxB,IAAI,SAAS,SAAS,MACpB,wBAAwB,EAAE;CAG5B,MAAM,QAAQ,eAAe,MAAM;CAInC,MAAM,WAAW,qBACb,MAAM,gBAAgB,IAAI,OAAO,IAAI,KAAK,IAC1C;CAEJ,IAAI,YAAY;CAChB,IAAI;CACJ,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,MAAM,eAAe,IAAI,mBAAmB;CAC5C,IAAI,kCAAkC;CACtC,IAAI,mBAAmB;CACvB,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAAG;EAGjC,MAAM,UAAU,IAAI,YAAY,EAAE;EAyBlC,MAAM,iBAAiB;GAAC;GAAY,kBAjBlC,YAAY,WAAW,aAAa,WAAW,aAC3C,OAAO,kBAA2B;IAChC,MAAM,OAAO,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,CAAC,CAAC,QACtD,aACF;IACA,YAAY,SAAS,gBAAgB,OAAO;IAC5C,IAAI,WAAW;KACb,MAAM,WAAW;KACjB,QAAQ,UAAU,oBAAoB;MACpC,YAAY;MACZ,OAAO,SAAS,QAAQ;KAC1B,CAAC;IACH;IACA,OAAO;GACT,IACA,KAAA;GAEgD;EAAO;EAC7D,IAAI;GAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;GACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,4BACA,kBAAkB,sBAClB,uBACA,iBACE,SAAS,SAAS,OAClB,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;IACE,GAAG;IACH,yBAAyB,SAAS;IAClC,6BAA6B,SAAS;GACxC,GACA,SACA,eAAe,kBACf,gBACA;IACE;IACA;IACA;IACA,UAAU,WAAW;IACrB,aAAa;GACf,GACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;GAEJ,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;GAMxD,gBAAgB,2BAA2B;IACzC;IACA,QAAQ,uBACJ;KACE,GAAG;KACH,YAAY;MACV,GAAG,OAAO;MACV,mBAAmB;KACrB;IACF,IACA;IACJ,aAAa,QAAQ;GACvB,CAAC;GAED,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;IACE,WAAW;IACX,mBAAmB,WAAW,gBAAgB,OAAO;IACrD,aAAa,QAAQ;GACvB,IACA,MACJ,WACA,aAAa,OACb;IACE,uBAAuB,yBAAyB;IAChD;IACA;IACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;IACN;IAIA,0CACG,eAAe,mBAAmB,IAC/B,eAAe,mBAAmB,MAClC,KAAK;IACX,oBAAoB,OAAO;IAC3B,oBAAoB,OAAO;IAC3B,eAAe;KACb,eAAe,kBAAkB,QAAQ,IAAI;KAC7C,eAAe;KACf,kBAAkB;IACpB;IAIA,uBAAuB,yBACnB;KACE,aAAa,QAAQ;KACrB,UAAU;MAGR,OAAO;MACP,YAAY,kBAAkB;MAC9B;KACF;IACF,IACA,KAAA;IAIJ,sBAAsB,yBAClB;KACE,aAAa;KACb,WAAW;IACb,IACA,KAAA;IAEJ,wBACE,sBAAsB,2BAClB;KACE,aAAa,QAAQ;KACrB,gBAAgB;KAChB,aAAa;KACb,eAAe;IACjB,IACA,KAAA;IAIN,sBACE,wBAAwB,UACpB;KACE,aAAa;KACb,iBAAiB;KACjB,aAAa;IACf,IACA,KAAA;IACN;GACF,GACA,UACF;GACA,YAAY;GACZ,kCAAkC;GAClC,aAAa,QAAQ,cAAc,IAAI;GACvC,aAAa,OAAO,eAAe,cAAc,YAAY;GAC7D,aAAa,OAAO,aAAa,cAAc,UAAU;GACzD,aAAa,OAAO,WAAW,cAAc,SAAS;GACtD,mBAAmB;GACnB;EACF,SAAS,GAAG;GAcV,MAAM,QAAQ,QAAQ;GAEtB,eAAoB,KAAK;GACzB,gBAAgB,KAAA;GAChB,IAAI,SAAS,aAAa,iBAAiB;IACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;IAGhD,aAAa,QAAQ,IAAI;IAMzB,kBAAkB,QAAQ,IAAI;IAC9B;GACF;GACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;GAEF,MAAM;EACR;CACF;CAEA,OAAO,gBAAgB,kDAAkD;CACzE,OAAO,eAAe,oDAAoD;CAI1E,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,MAAM,oBAA8B,CAAC;CACrC,MAAM,EAAC,SAAS,iBAAiB,YAAW,SAAS;CACrD,IAAI,WAAW;EACb,MAAM,eAAe,YAAY,IAAI;EACrC,gBAAqB,WACnB,aAAa,OAAO,YAAY,YAAY,IAAI,IAAI,YAAY,CAClE;CACF;CACA,IAAI,CAAC,WACH,QAAQ;MACH;EACL,GAAG,OAAO,wBAAwB,SAAS;EAC3C,OAAO,WAAW,oBAAoB;EACtC,IAAI,kBACF,wBAAwB,QAAQ,IAAI;EAItC,MAAM,mBAAmB,UACtB,IACC,YACE,gBACA;GAAC,GAAG;GAAK,qCAAqC;EAAS,GACvD,QACF,GACA,mBACF,CAAC,CACA,gBAAgB,eAAe,QAAQ,CAAC;EAC3C,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;CACF;CAEA,IAAI,cAAc;CAClB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAc,cAAc,oBAAoB,CAAC,CAAC,WAAW;GAC3D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;GAClE,aAAa,OAAO,kBAAkB,OAAO;EAC/C,CAAC;CACH;CAOA,MAAM,gBAAgB,qBAClB,cACA,QAAQ,IAAI,CAAC,aAAa,eAAe,CAAC;CAK9C,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAGA,gBAAgB,qBAAqB,iBAAiB;EACtD;EACA;EACA,kBAdqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAaF,GACA,QACA,cACF;CAEA,cAAmB,WAAW;EAC5B,aAAa,MAAM;EACnB,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CASD,UAAU;CACV,MAAM,eACJ,IACA,QACA,yBACA,eACA,SACF;AACF;;;;;;;AAQA,eAAe,gBACb,IACA,eACA,OACqB;CACrB,MAAM,WAAW,MAAM,gBACrB,IACA,eACA,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC;CAC7D,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,OAAO;AACT;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,CAAC,CAAC,MAAM,OAAM,MAAK;CACjB,IAAI,CAAC,SACH,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CAEF,MAAM;AACR,CAAC,CACL"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {resolver} from '../../../shared/src/resolver.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {InitCleanup} from '../services/change-source/common/init-cleanup.ts';\nimport type {PgChangeLogPurgeLocker} from '../services/change-source/common/replica-restore.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport type {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {DEFAULT_READINESS_GRACE_PERIOD_MS} from '../services/change-streamer/restore-admission.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {ServerStartMetrics} from '../services/change-streamer/server-start-metrics.ts';\nimport {\n  PurgeLocker,\n  type PurgeLock,\n} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {deleteLitestreamMetaDir} from '../services/litestream/commands.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {warnIfPostgresUnpatched} from '../types/pg-patch.ts';\nimport {connectPgClient, type PostgresDB} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig, type ShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\n// Errors thrown once the services are running are reported by the services\n// themselves (e.g. as 'Replicating' status events), and are thus not\n// published as 'Initializing' errors.\nlet running = false;\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      maxConcurrentSwitchoverRestores,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n  if (upstream.type === 'pg') {\n    warnIfPostgresUnpatched(lc);\n  }\n\n  const shard = getShardConfig(config);\n\n  // The change DB is only used by the PG change log. When it is disabled\n  // (RMv2), no connection to the change DB is made.\n  const changeDB = pgChangeLogEnabled\n    ? await connectChangeDB(lc, change.db, shard)\n    : null;\n\n  let purgeLock = null as PurgeLock | null;\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupMonitor: BackupMonitor | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  const startMetrics = new ServerStartMetrics();\n  let waitForFirstBackupBeforeServing = false;\n  let newBackupLineage = false;\n  for (const first of [true, false]) {\n    // Resources acquired by an initialization attempt (e.g. a claimed\n    // replication slot, the purge lock) to be released if the attempt fails.\n    const cleanup = new InitCleanup(lc);\n    // When restoring from litestream, a lock is acquired to prevent change-log\n    // purges. This ensures that (this) change-streamer will be able to resume\n    // from the backup. The lock is acquired by the change source initialization\n    // just before restoring (i.e. after a replication slot is created, if\n    // applicable), and released once this change-streamer takes over the\n    // change-log.\n    const acquirePurgeLock: PgChangeLogPurgeLocker | undefined =\n      changeDB && litestream.backupURL && litestream.executable\n        ? async (slotWatermark?: string) => {\n            const lock = await new PurgeLocker(lc, shard, changeDB).acquire(\n              slotWatermark,\n            );\n            purgeLock = lock === 'behind-slot' ? null : lock;\n            if (purgeLock) {\n              const acquired = purgeLock;\n              cleanup.onFailure('purge lock', () => {\n                purgeLock = null;\n                return acquired.release();\n              });\n            }\n            return lock;\n          }\n        : undefined;\n\n    const restoreOptions = {litestream, acquirePurgeLock, cleanup};\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        pgReplicationEpoch: epoch,\n        pgReplicationSlotPerReplica: slotPerReplica,\n        pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n      } = upstream;\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n        newBackupLineage: initNewBackupLineage,\n        pgChangeLogBehindSlot,\n        startTimings,\n      } = upstream.type === 'pg'\n        ? await initializePostgresChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            {\n              ...initialSync,\n              replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n            },\n            context,\n            replicationLag.reportIntervalMs,\n            restoreOptions,\n            {\n              epoch,\n              slotPerReplica,\n              inactiveReplicaGracePeriodMs,\n              backupV5: litestream.backupUsingV5,\n              pgChangeLog: pgChangeLogEnabled,\n            },\n            upstream.pgStreamInboundTimeoutMs,\n          )\n        : await initializeCustomChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            context,\n            restoreOptions,\n          );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      // Created before the change-streamer, which tracks its watermarks (and\n      // requests prompt backups from it) from construction. It is run (and\n      // stopped) as a service below. It monitors the backup-replicator's\n      // destination (see `backupURLOverride` below).\n      backupMonitor = createBackupCleanupMonitor({\n        lc,\n        config: destinationBackupURL\n          ? {\n              ...config,\n              litestream: {\n                ...config.litestream,\n                backupURLOverride: destinationBackupURL,\n              },\n            }\n          : config,\n        replicaFile: replica.file,\n      });\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n              replicaFile: replica.file,\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogBehindSlot: pgChangeLogBehindSlot ?? false,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          maxConcurrentSwitchoverRestores,\n          // When lag reporting is enabled, a restored view-syncer waits for\n          // the next lag report (sent at an interval with ~1 second of\n          // jitter) before becoming ready.\n          switchoverRestoreReadinessGracePeriodMs:\n            (replicationLag.reportIntervalMs > 0\n              ? replicationLag.reportIntervalMs + 1000\n              : 0) + DEFAULT_READINESS_GRACE_PERIOD_MS,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n          backupMonitor,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      startMetrics.setKind(startTimings?.kind);\n      startMetrics.record('slot_create', startTimings?.slotCreateMs);\n      startMetrics.record('fork_wait', startTimings?.forkWaitMs);\n      startMetrics.record('restore', startTimings?.restoreMs);\n      newBackupLineage = initNewBackupLineage;\n      break;\n    } catch (e) {\n      // Release the resources acquired by the failed attempt, e.g.:\n      // * The claimed replication slot, which would otherwise remain active\n      //   indefinitely, preventing a retry from resuming (or forking) it.\n      // * The purge lock. This is safe because the purge lock exists to\n      //   preserve change-log entries so the new change-streamer can resume\n      //   from the backup replica's watermark. An AutoResetSignal means we\n      //   can't resume from the backup replica (e.g. its replication slot is\n      //   gone), so the change-log entries the lock was protecting are no\n      //   longer needed. The retry performs a fresh initial sync with a new\n      //   replication slot, independent of the old change-log. Releasing is\n      //   also necessary to avoid a self-deadlock when CHANGE_DB ==\n      //   UPSTREAM_DB: CREATE_REPLICATION_SLOT waits for all older\n      //   transactions to finish, including this lock's open transaction.\n      await cleanup.release();\n      // Not yet started (the change-streamer starts it), so nothing to wait on.\n      void backupMonitor?.stop();\n      backupMonitor = undefined;\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n  assert(backupMonitor, `backup monitor is created with the change-streamer`);\n\n  // The backup-replicator is stopped with the other services, i.e. after the\n  // ChangeStreamerHttpServer has drained its subscribers.\n  const processes = new SupportingProcesses(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  const {promise: replicatorReady, resolve} = resolver();\n  if (backupURL) {\n    const catchUpStart = performance.now();\n    void replicatorReady.then(() =>\n      startMetrics.record('catch_up', performance.now() - catchUpStart),\n    );\n  }\n  if (!backupURL) {\n    resolve(); // No backup replicator to wait for, so resolve immediately.\n  } else {\n    lc.info?.('setting up backup to', backupURL);\n    config.litestream.backupURLOverride = backupURL;\n    if (newBackupLineage) {\n      deleteLitestreamMetaDir(replica.file);\n    }\n    // Start a backup replicator, which starts up the corresponding\n    // litestream backup process.\n    const backupReplicator = processes\n      .add(\n        childWorker(\n          REPLICATOR_URL,\n          {...env, ZERO_LITESTREAM_BACKUP_URL_OVERRIDE: backupURL},\n          'backup' satisfies ReplicaFileMode,\n        ),\n        'backup-replicator',\n      )\n      .onceMessageType('ready', () => resolve());\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n  }\n\n  let backupReady = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    backupReady = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n      startMetrics.record('backup_confirm', elapsed);\n    });\n  }\n\n  // In RMv2, readiness additionally waits for the backup-replicator to catch\n  // up to the replication stream. This is not done in RMv1 (i.e. when the\n  // PG change-log is enabled), in which the change-streamer itself is only\n  // started after the readinessGate (plus a startup delay), and thus the\n  // backup-replicator cannot catch up before then.\n  const readinessGate = pgChangeLogEnabled\n    ? backupReady\n    : Promise.all([backupReady, replicatorReady]);\n\n  // Create the broadcast facade once: each broadcastWorker() adds permanent\n  // 'message' forwarders to every sub worker, so creating one per /profz\n  // request would leak a forwarder per request.\n  const profileWorker =\n    profileSubWorkers.length > 0\n      ? broadcastWorker(profileSubWorkers)\n      : undefined;\n  const getProfileWorker = profileWorker\n    ? () => Promise.resolve(profileWorker)\n    : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      // The startup delay is only relevant when taking over the PG change-log\n      // (RMv1 and RMv1.5), and is disabled for RMv2.\n      startupDelayMs: pgChangeLogEnabled ? startupDelayMs : 0,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => {\n    startMetrics.ready();\n    parent.send(['ready', {ready: true}]);\n  });\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  running = true;\n  await runUntilKilled(\n    lc,\n    parent,\n    changeStreamerWebServer,\n    backupMonitor,\n    processes,\n  );\n}\n\n/**\n * Connects the startup-time client used for change-streamer initialization\n * and handoff / takeover, and ensures that the change DB schema is\n * initialized / up-to-date. Steady-state clients are managed by the\n * change-streamer (Storer) implementation.\n */\nasync function connectChangeDB(\n  lc: LogContext,\n  connectionURI: string,\n  shard: ShardConfig,\n): Promise<PostgresDB> {\n  const changeDB = await connectPgClient(\n    lc,\n    connectionURI,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  return changeDB;\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        if (!running) {\n          await publishCriticalEvent(\n            lc,\n            replicationStatusError(lc, 'Initializing', e),\n          );\n        }\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAKlD,IAAI,UAAU;AAEd,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,iCACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CACxB,IAAI,SAAS,SAAS,MACpB,wBAAwB,EAAE;CAG5B,MAAM,QAAQ,eAAe,MAAM;CAInC,MAAM,WAAW,qBACb,MAAM,gBAAgB,IAAI,OAAO,IAAI,KAAK,IAC1C;CAEJ,IAAI,YAAY;CAChB,IAAI;CACJ,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,MAAM,eAAe,IAAI,mBAAmB;CAC5C,IAAI,kCAAkC;CACtC,IAAI,mBAAmB;CACvB,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAAG;EAGjC,MAAM,UAAU,IAAI,YAAY,EAAE;EAyBlC,MAAM,iBAAiB;GAAC;GAAY,kBAjBlC,YAAY,WAAW,aAAa,WAAW,aAC3C,OAAO,kBAA2B;IAChC,MAAM,OAAO,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,CAAC,CAAC,QACtD,aACF;IACA,YAAY,SAAS,gBAAgB,OAAO;IAC5C,IAAI,WAAW;KACb,MAAM,WAAW;KACjB,QAAQ,UAAU,oBAAoB;MACpC,YAAY;MACZ,OAAO,SAAS,QAAQ;KAC1B,CAAC;IACH;IACA,OAAO;GACT,IACA,KAAA;GAEgD;EAAO;EAC7D,IAAI;GAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;GACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,4BACA,kBAAkB,sBAClB,uBACA,iBACE,SAAS,SAAS,OAClB,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;IACE,GAAG;IACH,yBAAyB,SAAS;IAClC,6BAA6B,SAAS;GACxC,GACA,SACA,eAAe,kBACf,gBACA;IACE;IACA;IACA;IACA,UAAU,WAAW;IACrB,aAAa;GACf,GACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;GAEJ,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;GAMxD,gBAAgB,2BAA2B;IACzC;IACA,QAAQ,uBACJ;KACE,GAAG;KACH,YAAY;MACV,GAAG,OAAO;MACV,mBAAmB;KACrB;IACF,IACA;IACJ,aAAa,QAAQ;GACvB,CAAC;GAED,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;IACE,WAAW;IACX,mBAAmB,WAAW,gBAAgB,OAAO;IACrD,aAAa,QAAQ;GACvB,IACA,MACJ,WACA,aAAa,OACb;IACE,uBAAuB,yBAAyB;IAChD;IACA;IACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;IACN;IAIA,0CACG,eAAe,mBAAmB,IAC/B,eAAe,mBAAmB,MAClC,KAAK;IACX,oBAAoB,OAAO;IAC3B,oBAAoB,OAAO;IAC3B,eAAe;KACb,eAAe,kBAAkB,QAAQ,IAAI;KAC7C,eAAe;KACf,kBAAkB;IACpB;IAIA,uBAAuB,yBACnB;KACE,aAAa,QAAQ;KACrB,UAAU;MAGR,OAAO;MACP,YAAY,kBAAkB;MAC9B;KACF;IACF,IACA,KAAA;IAIJ,sBAAsB,yBAClB;KACE,aAAa;KACb,WAAW;IACb,IACA,KAAA;IAEJ,wBACE,sBAAsB,2BAClB;KACE,aAAa,QAAQ;KACrB,gBAAgB;KAChB,aAAa;KACb,eAAe;IACjB,IACA,KAAA;IAIN,sBACE,wBAAwB,UACpB;KACE,aAAa;KACb,iBAAiB;KACjB,aAAa;IACf,IACA,KAAA;IACN;GACF,GACA,UACF;GACA,YAAY;GACZ,kCAAkC;GAClC,aAAa,QAAQ,cAAc,IAAI;GACvC,aAAa,OAAO,eAAe,cAAc,YAAY;GAC7D,aAAa,OAAO,aAAa,cAAc,UAAU;GACzD,aAAa,OAAO,WAAW,cAAc,SAAS;GACtD,mBAAmB;GACnB;EACF,SAAS,GAAG;GAcV,MAAM,QAAQ,QAAQ;GAEtB,eAAoB,KAAK;GACzB,gBAAgB,KAAA;GAChB,IAAI,SAAS,aAAa,iBAAiB;IACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;IAGhD,aAAa,QAAQ,IAAI;IAMzB,kBAAkB,QAAQ,IAAI;IAC9B;GACF;GACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;GAEF,MAAM;EACR;CACF;CAEA,OAAO,gBAAgB,kDAAkD;CACzE,OAAO,eAAe,oDAAoD;CAI1E,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,MAAM,oBAA8B,CAAC;CACrC,MAAM,EAAC,SAAS,iBAAiB,YAAW,SAAS;CACrD,IAAI,WAAW;EACb,MAAM,eAAe,YAAY,IAAI;EACrC,gBAAqB,WACnB,aAAa,OAAO,YAAY,YAAY,IAAI,IAAI,YAAY,CAClE;CACF;CACA,IAAI,CAAC,WACH,QAAQ;MACH;EACL,GAAG,OAAO,wBAAwB,SAAS;EAC3C,OAAO,WAAW,oBAAoB;EACtC,IAAI,kBACF,wBAAwB,QAAQ,IAAI;EAItC,MAAM,mBAAmB,UACtB,IACC,YACE,gBACA;GAAC,GAAG;GAAK,qCAAqC;EAAS,GACvD,QACF,GACA,mBACF,CAAC,CACA,gBAAgB,eAAe,QAAQ,CAAC;EAC3C,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;CACF;CAEA,IAAI,cAAc;CAClB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAc,cAAc,oBAAoB,CAAC,CAAC,WAAW;GAC3D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;GAClE,aAAa,OAAO,kBAAkB,OAAO;EAC/C,CAAC;CACH;CAOA,MAAM,gBAAgB,qBAClB,cACA,QAAQ,IAAI,CAAC,aAAa,eAAe,CAAC;CAK9C,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAGA,gBAAgB,qBAAqB,iBAAiB;EACtD;EACA;EACA,kBAdqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAaF,GACA,QACA,cACF;CAEA,cAAmB,WAAW;EAC5B,aAAa,MAAM;EACnB,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CASD,UAAU;CACV,MAAM,eACJ,IACA,QACA,yBACA,eACA,SACF;AACF;;;;;;;AAQA,eAAe,gBACb,IACA,eACA,OACqB;CACrB,MAAM,WAAW,MAAM,gBACrB,IACA,eACA,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC;CAC7D,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,OAAO;AACT;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,CAAC,CAAC,MAAM,OAAM,MAAK;CACjB,IAAI,CAAC,SACH,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CAEF,MAAM;AACR,CAAC,CACL"}

package/out/zero-cache/src/server/main.js

@@ -1,3 +1,4 @@
+import { resolver } from "../../../shared/src/resolver.js";
 import { must } from "../../../shared/src/must.js";
 import { childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
 import { runsChangeStreamer } from "../config/normalize.js";
@@ -10,7 +11,6 @@ import { initEventSink } from "../observability/events.js";
 import { startOtelAuto } from "./otel-start.js";
 import { createNotifierFrom, handleSubscriptionsFrom, subscribeTo } from "../workers/replicator.js";
 import { WorkerDispatcher } from "./worker-dispatcher.js";
-import { resolver } from "@rocicorp/resolver";
 import { LogContext, consoleLogSink } from "@rocicorp/logger";
 import path from "node:path";
 //#region ../zero-cache/src/server/main.ts

package/out/zero-cache/src/server/main.js.map

@@ -1 +1 @@
-{"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../shared/src/must.ts';\nimport {runsChangeStreamer} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  recordStartupDurationMs,\n  runUntilKilled,\n  type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type DrainRequestedMessage,\n  type ProfileMessage,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {\n  createNotifierFrom,\n  handleSubscriptionsFrom,\n  subscribeTo,\n  type ReplicaFileMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n  CHANGE_STREAMER_URL,\n  MUTATOR_URL,\n  REAPER_URL,\n  REPLICATOR_URL,\n  SHADOW_SYNCER_URL,\n  SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  const startMs = Date.now();\n  const config = getNormalizedZeroConfig({env});\n\n  startOtelAuto(\n    createLogContext(config, 'dispatcher', 0, false),\n    'dispatcher',\n    0,\n  );\n  lc = createLogContext(config, 'dispatcher');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'dispatcher replica',\n      dbPath: config.replica.file,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  const processes = new ProcessManager(lc, parent);\n\n  const {numSyncWorkers: numSyncers} = config;\n  if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n  if (config.cvr.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n\n  const internalFlags: string[] =\n    numSyncers === 0\n      ? []\n      : [\n          '--upstream-max-conns-per-worker',\n          String(Math.floor(config.upstream.maxConns / numSyncers)),\n          '--cvr-max-conns-per-worker',\n          String(Math.floor(config.cvr.maxConns / numSyncers)),\n        ];\n\n  const allSubWorkers: Worker[] = [];\n  function loadWorker(\n    moduleUrl: URL,\n    type: WorkerType,\n    id?: string | number,\n    ...args: string[]\n  ): Worker {\n    const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n    const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n    const w = processes.addWorker(worker, type, name);\n    allSubWorkers.push(w);\n    w.onMessageType<ProfileResponseMessage>('profileResponse', res =>\n      parent.send(['profileResponse', res]),\n    );\n    return w;\n  }\n\n  const {taskID, litestream} = config;\n  const runChangeStreamer = runsChangeStreamer(config);\n  const sqliteChangeLogEnabled =\n    config.changeStreamer.sqliteChangeLogMode !== 'off';\n  if (sqliteChangeLogEnabled && !runChangeStreamer) {\n    // A multi-node deployment configures every task from one environment, so\n    // the change log's options reach the view-syncers too. The log lives in\n    // the change-streamer, and a task that connects to one simply never writes\n    // a log. Warning rather than refusing to start is what makes the\n    // fleet-wide environment -- often the only knob an operator has -- able to\n    // express the rollout at all.\n    lc.warn?.(\n      `ignoring --change-streamer-sqlite-change-log-mode=` +\n        `${config.changeStreamer.sqliteChangeLogMode}: this task connects to ` +\n        `a change-streamer rather than running one, and the SQLite change log ` +\n        `lives in the change-streamer`,\n    );\n  }\n\n  let changeStreamer: Worker | undefined;\n\n  if (runChangeStreamer) {\n    const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n      resolver();\n    // In a dedicated replication-manager (i.e. no syncers), the\n    // change-streamer is the user-facing worker: it receives the SIGTERM\n    // first so that it can drain subscribers before other supporting\n    // workers are stopped. When syncers run in the same task, the\n    // change-streamer supports them and thus must outlive them.\n    changeStreamer = loadWorker(\n      CHANGE_STREAMER_URL,\n      numSyncers > 0 ? 'supporting' : 'user-facing',\n    ).once('message', changeStreamerStarted);\n    // Wait for the change-streamer to be ready to guarantee that a replica\n    // file is present.\n    await changeStreamerReady;\n  }\n\n  if (numSyncers > 0) {\n    const {promise: reaperReady, resolve: reaperStarted} = resolver();\n    loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n    // Before starting the view-syncers, ensure that the reaper has started\n    // up, indicating that any CVR db migrations have been performed.\n    await reaperReady;\n  }\n\n  // Only run the shadow-sync canary on the replication-manager (or in\n  // single-node mode, where it also owns upstream). Running on every\n  // view-syncer would hammer the upstream with N redundant canaries.\n  if (config.shadowSync.enabled && runChangeStreamer) {\n    const {promise: shadowReady, resolve: shadowStarted} = resolver();\n    loadWorker(SHADOW_SYNCER_URL, 'supporting').once('message', shadowStarted);\n    await shadowReady;\n  }\n\n  const syncers: Worker[] = [];\n  if (numSyncers) {\n    const mode: ReplicaFileMode =\n      runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n    const {promise: replicaReady, resolve} = resolver();\n    const replicator = loadWorker(\n      REPLICATOR_URL,\n      'supporting',\n      mode,\n      mode,\n    ).once('message', () => {\n      subscribeTo(lc, replicator);\n      resolve();\n    });\n    // Relay drain requests to the runner, which coordinates the SelfDrain.\n    replicator.onMessageType<DrainRequestedMessage>('drainRequested', msg => {\n      lc.info?.(`drain requested by the replicator`);\n      parent.send<DrainRequestedMessage>(['drainRequested', msg]);\n    });\n    await replicaReady;\n\n    const notifier = createNotifierFrom(lc, replicator);\n    for (let i = 0; i < numSyncers; i++) {\n      syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n    }\n    syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n  }\n  let mutator: Worker | undefined;\n  if (clientConnectionBifurcated) {\n    mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n  }\n\n  lc.info?.('waiting for workers to be ready ...');\n  const logWaiting = setInterval(\n    () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n    10_000,\n  );\n  await processes.allWorkersReady();\n  clearInterval(logWaiting);\n  const startupDurationMs = Date.now() - startMs;\n  lc.info?.(`all workers ready (${startupDurationMs} ms)`);\n  recordStartupDurationMs(startupDurationMs);\n\n  parent.send(['ready', {ready: true}]);\n  parent.onMessageType<ProfileMessage>('profile', req => {\n    for (const w of allSubWorkers) {\n      w.send(['profile', req]);\n    }\n  });\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent,\n      new WorkerDispatcher(\n        lc,\n        taskID,\n        parent,\n        syncers,\n        mutator,\n        changeStreamer,\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'dispatcher');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA6CA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,wBAAwB,EAAC,IAAG,CAAC;CAE5C,cACE,iBAAiB,QAAQ,cAAc,GAAG,KAAK,GAC/C,cACA,CACF;CACA,KAAK,iBAAiB,QAAQ,YAAY;CAC1C,yCACE;EACE,WAAW;EACX,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACA,cAAc,IAAI,MAAM;CAExB,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAE/C,MAAM,EAAC,gBAAgB,eAAc;CACrC,IAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,YAC3D,MAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,iHAEpF;CAEF,IAAI,OAAO,IAAI,WAAW,YACxB,MAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,4GAE1E;CAGF,MAAM,gBACJ,eAAe,IACX,CAAC,IACD;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,UAAU,CAAC;EACxD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,UAAU,CAAC;CACrD;CAEN,MAAM,gBAA0B,CAAC;CACjC,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,aAAa;EACpE,MAAM,OAAO,KAAK,SAAS,UAAU,QAAQ,KAAK,KAAK,KAAK,GAAG,KAAK;EACpE,MAAM,IAAI,UAAU,UAAU,QAAQ,MAAM,IAAI;EAChD,cAAc,KAAK,CAAC;EACpB,EAAE,cAAsC,oBAAmB,QACzD,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CACtC;EACA,OAAO;CACT;CAEA,MAAM,EAAC,QAAQ,eAAc;CAC7B,MAAM,oBAAoB,mBAAmB,MAAM;CAGnD,IADE,OAAO,eAAe,wBAAwB,SAClB,CAAC,mBAO7B,GAAG,OACD,qDACK,OAAO,eAAe,oBAAoB,0HAGjD;CAGF,IAAI;CAEJ,IAAI,mBAAmB;EACrB,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,SAAS;EAMX,iBAAiB,WACf,qBACA,aAAa,IAAI,eAAe,aAClC,CAAC,CAAC,KAAK,WAAW,qBAAqB;EAGvC,MAAM;CACR;CAEA,IAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,YAAY,YAAY,CAAC,CAAC,KAAK,WAAW,aAAa;EAGlE,MAAM;CACR;CAKA,IAAI,OAAO,WAAW,WAAW,mBAAmB;EAClD,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,mBAAmB,YAAY,CAAC,CAAC,KAAK,WAAW,aAAa;EACzE,MAAM;CACR;CAEA,MAAM,UAAoB,CAAC;CAC3B,IAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,SAAS;EAClD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,IACF,CAAC,CAAC,KAAK,iBAAiB;GACtB,YAAY,IAAI,UAAU;GAC1B,QAAQ;EACV,CAAC;EAED,WAAW,cAAqC,mBAAkB,QAAO;GACvE,GAAG,OAAO,mCAAmC;GAC7C,OAAO,KAA4B,CAAC,kBAAkB,GAAG,CAAC;EAC5D,CAAC;EACD,MAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,UAAU;EAClD,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAC9B,QAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC;EAExE,QAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,QAAQ,CAAC;CACzE;CACA,IAAI;CAKJ,GAAG,OAAO,qCAAqC;CAC/C,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,aAAa,CAAC,CAAC,KAAK,IAAI,GAAG,GAC1E,GACF;CACA,MAAM,UAAU,gBAAgB;CAChC,cAAc,UAAU;CACxB,MAAM,oBAAoB,KAAK,IAAI,IAAI;CACvC,GAAG,OAAO,sBAAsB,kBAAkB,KAAK;CACvD,wBAAwB,iBAAiB;CAEzC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACpC,OAAO,cAA8B,YAAW,QAAO;EACrD,KAAK,MAAM,KAAK,eACd,EAAE,KAAK,CAAC,WAAW,GAAG,CAAC;CAE3B,CAAC;CAED,IAAI;EACF,MAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,cACF,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,YAAY;CAC7C,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,GAAG,CACjD"}
+{"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {resolver} from '../../../shared/src/resolver.ts';\nimport {runsChangeStreamer} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  recordStartupDurationMs,\n  runUntilKilled,\n  type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type DrainRequestedMessage,\n  type ProfileMessage,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {\n  createNotifierFrom,\n  handleSubscriptionsFrom,\n  subscribeTo,\n  type ReplicaFileMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n  CHANGE_STREAMER_URL,\n  MUTATOR_URL,\n  REAPER_URL,\n  REPLICATOR_URL,\n  SHADOW_SYNCER_URL,\n  SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  const startMs = Date.now();\n  const config = getNormalizedZeroConfig({env});\n\n  startOtelAuto(\n    createLogContext(config, 'dispatcher', 0, false),\n    'dispatcher',\n    0,\n  );\n  lc = createLogContext(config, 'dispatcher');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'dispatcher replica',\n      dbPath: config.replica.file,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  const processes = new ProcessManager(lc, parent);\n\n  const {numSyncWorkers: numSyncers} = config;\n  if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n  if (config.cvr.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n\n  const internalFlags: string[] =\n    numSyncers === 0\n      ? []\n      : [\n          '--upstream-max-conns-per-worker',\n          String(Math.floor(config.upstream.maxConns / numSyncers)),\n          '--cvr-max-conns-per-worker',\n          String(Math.floor(config.cvr.maxConns / numSyncers)),\n        ];\n\n  const allSubWorkers: Worker[] = [];\n  function loadWorker(\n    moduleUrl: URL,\n    type: WorkerType,\n    id?: string | number,\n    ...args: string[]\n  ): Worker {\n    const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n    const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n    const w = processes.addWorker(worker, type, name);\n    allSubWorkers.push(w);\n    w.onMessageType<ProfileResponseMessage>('profileResponse', res =>\n      parent.send(['profileResponse', res]),\n    );\n    return w;\n  }\n\n  const {taskID, litestream} = config;\n  const runChangeStreamer = runsChangeStreamer(config);\n  const sqliteChangeLogEnabled =\n    config.changeStreamer.sqliteChangeLogMode !== 'off';\n  if (sqliteChangeLogEnabled && !runChangeStreamer) {\n    // A multi-node deployment configures every task from one environment, so\n    // the change log's options reach the view-syncers too. The log lives in\n    // the change-streamer, and a task that connects to one simply never writes\n    // a log. Warning rather than refusing to start is what makes the\n    // fleet-wide environment -- often the only knob an operator has -- able to\n    // express the rollout at all.\n    lc.warn?.(\n      `ignoring --change-streamer-sqlite-change-log-mode=` +\n        `${config.changeStreamer.sqliteChangeLogMode}: this task connects to ` +\n        `a change-streamer rather than running one, and the SQLite change log ` +\n        `lives in the change-streamer`,\n    );\n  }\n\n  let changeStreamer: Worker | undefined;\n\n  if (runChangeStreamer) {\n    const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n      resolver();\n    // In a dedicated replication-manager (i.e. no syncers), the\n    // change-streamer is the user-facing worker: it receives the SIGTERM\n    // first so that it can drain subscribers before other supporting\n    // workers are stopped. When syncers run in the same task, the\n    // change-streamer supports them and thus must outlive them.\n    changeStreamer = loadWorker(\n      CHANGE_STREAMER_URL,\n      numSyncers > 0 ? 'supporting' : 'user-facing',\n    ).once('message', changeStreamerStarted);\n    // Wait for the change-streamer to be ready to guarantee that a replica\n    // file is present.\n    await changeStreamerReady;\n  }\n\n  if (numSyncers > 0) {\n    const {promise: reaperReady, resolve: reaperStarted} = resolver();\n    loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n    // Before starting the view-syncers, ensure that the reaper has started\n    // up, indicating that any CVR db migrations have been performed.\n    await reaperReady;\n  }\n\n  // Only run the shadow-sync canary on the replication-manager (or in\n  // single-node mode, where it also owns upstream). Running on every\n  // view-syncer would hammer the upstream with N redundant canaries.\n  if (config.shadowSync.enabled && runChangeStreamer) {\n    const {promise: shadowReady, resolve: shadowStarted} = resolver();\n    loadWorker(SHADOW_SYNCER_URL, 'supporting').once('message', shadowStarted);\n    await shadowReady;\n  }\n\n  const syncers: Worker[] = [];\n  if (numSyncers) {\n    const mode: ReplicaFileMode =\n      runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n    const {promise: replicaReady, resolve} = resolver();\n    const replicator = loadWorker(\n      REPLICATOR_URL,\n      'supporting',\n      mode,\n      mode,\n    ).once('message', () => {\n      subscribeTo(lc, replicator);\n      resolve();\n    });\n    // Relay drain requests to the runner, which coordinates the SelfDrain.\n    replicator.onMessageType<DrainRequestedMessage>('drainRequested', msg => {\n      lc.info?.(`drain requested by the replicator`);\n      parent.send<DrainRequestedMessage>(['drainRequested', msg]);\n    });\n    await replicaReady;\n\n    const notifier = createNotifierFrom(lc, replicator);\n    for (let i = 0; i < numSyncers; i++) {\n      syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n    }\n    syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n  }\n  let mutator: Worker | undefined;\n  if (clientConnectionBifurcated) {\n    mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n  }\n\n  lc.info?.('waiting for workers to be ready ...');\n  const logWaiting = setInterval(\n    () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n    10_000,\n  );\n  await processes.allWorkersReady();\n  clearInterval(logWaiting);\n  const startupDurationMs = Date.now() - startMs;\n  lc.info?.(`all workers ready (${startupDurationMs} ms)`);\n  recordStartupDurationMs(startupDurationMs);\n\n  parent.send(['ready', {ready: true}]);\n  parent.onMessageType<ProfileMessage>('profile', req => {\n    for (const w of allSubWorkers) {\n      w.send(['profile', req]);\n    }\n  });\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent,\n      new WorkerDispatcher(\n        lc,\n        taskID,\n        parent,\n        syncers,\n        mutator,\n        changeStreamer,\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'dispatcher');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA6CA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,wBAAwB,EAAC,IAAG,CAAC;CAE5C,cACE,iBAAiB,QAAQ,cAAc,GAAG,KAAK,GAC/C,cACA,CACF;CACA,KAAK,iBAAiB,QAAQ,YAAY;CAC1C,yCACE;EACE,WAAW;EACX,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACA,cAAc,IAAI,MAAM;CAExB,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAE/C,MAAM,EAAC,gBAAgB,eAAc;CACrC,IAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,YAC3D,MAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,iHAEpF;CAEF,IAAI,OAAO,IAAI,WAAW,YACxB,MAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,4GAE1E;CAGF,MAAM,gBACJ,eAAe,IACX,CAAC,IACD;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,UAAU,CAAC;EACxD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,UAAU,CAAC;CACrD;CAEN,MAAM,gBAA0B,CAAC;CACjC,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,aAAa;EACpE,MAAM,OAAO,KAAK,SAAS,UAAU,QAAQ,KAAK,KAAK,KAAK,GAAG,KAAK;EACpE,MAAM,IAAI,UAAU,UAAU,QAAQ,MAAM,IAAI;EAChD,cAAc,KAAK,CAAC;EACpB,EAAE,cAAsC,oBAAmB,QACzD,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CACtC;EACA,OAAO;CACT;CAEA,MAAM,EAAC,QAAQ,eAAc;CAC7B,MAAM,oBAAoB,mBAAmB,MAAM;CAGnD,IADE,OAAO,eAAe,wBAAwB,SAClB,CAAC,mBAO7B,GAAG,OACD,qDACK,OAAO,eAAe,oBAAoB,0HAGjD;CAGF,IAAI;CAEJ,IAAI,mBAAmB;EACrB,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,SAAS;EAMX,iBAAiB,WACf,qBACA,aAAa,IAAI,eAAe,aAClC,CAAC,CAAC,KAAK,WAAW,qBAAqB;EAGvC,MAAM;CACR;CAEA,IAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,YAAY,YAAY,CAAC,CAAC,KAAK,WAAW,aAAa;EAGlE,MAAM;CACR;CAKA,IAAI,OAAO,WAAW,WAAW,mBAAmB;EAClD,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,mBAAmB,YAAY,CAAC,CAAC,KAAK,WAAW,aAAa;EACzE,MAAM;CACR;CAEA,MAAM,UAAoB,CAAC;CAC3B,IAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,SAAS;EAClD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,IACF,CAAC,CAAC,KAAK,iBAAiB;GACtB,YAAY,IAAI,UAAU;GAC1B,QAAQ;EACV,CAAC;EAED,WAAW,cAAqC,mBAAkB,QAAO;GACvE,GAAG,OAAO,mCAAmC;GAC7C,OAAO,KAA4B,CAAC,kBAAkB,GAAG,CAAC;EAC5D,CAAC;EACD,MAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,UAAU;EAClD,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAC9B,QAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC;EAExE,QAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,QAAQ,CAAC;CACzE;CACA,IAAI;CAKJ,GAAG,OAAO,qCAAqC;CAC/C,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,aAAa,CAAC,CAAC,KAAK,IAAI,GAAG,GAC1E,GACF;CACA,MAAM,UAAU,gBAAgB;CAChC,cAAc,UAAU;CACxB,MAAM,oBAAoB,KAAK,IAAI,IAAI;CACvC,GAAG,OAAO,sBAAsB,kBAAkB,KAAK;CACvD,wBAAwB,iBAAiB;CAEzC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACpC,OAAO,cAA8B,YAAW,QAAO;EACrD,KAAK,MAAM,KAAK,eACd,EAAE,KAAK,CAAC,WAAW,GAAG,CAAC;CAE3B,CAAC;CAED,IAAI;EACF,MAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,cACF,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,YAAY;CAC7C,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,GAAG,CACjD"}

package/out/zero-cache/src/server/runner/run-worker.js

@@ -1,3 +1,4 @@
+import { resolver } from "../../../../shared/src/resolver.js";
 import "../../../../zero-protocol/src/protocol-version.js";
 import "../../../../shared/src/dotenv.js";
 import { colorConsole } from "../../../../shared/src/logging.js";
@@ -9,7 +10,6 @@ import { createLogContext } from "../logging.js";
 import { MAIN_URL } from "../worker-urls.js";
 import { getTaskID } from "./runtime.js";
 import { ZeroDispatcher } from "./zero-dispatcher.js";
-import { resolver } from "@rocicorp/resolver";
 import { styleText } from "node:util";
 //#region ../zero-cache/src/server/runner/run-worker.ts
 var startupMessageEnv = "ZERO_ENABLE_STARTUP_MESSAGE";

package/out/zero-cache/src/server/runner/run-worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-worker.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"sourcesContent":["import '../../../../shared/src/dotenv.ts';\n\nimport {styleText} from 'node:util';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {colorConsole} from '../../../../shared/src/logging.ts';\nimport {PROTOCOL_VERSION} from '../../../../zero-protocol/src/protocol-version.ts';\nimport {normalizeZeroConfig} from '../../config/normalize.ts';\nimport {getServerVersion, getZeroConfig} from '../../config/zero-config.ts';\nimport {\n  ProcessManager,\n  runUntilKilled,\n  SelfDrain,\n} from '../../services/life-cycle.ts';\nimport {\n  childWorker,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../../types/processes.ts';\nimport {createLogContext} from '../logging.ts';\nimport {MAIN_URL} from '../worker-urls.ts';\nimport {getTaskID} from './runtime.ts';\nimport {ZeroDispatcher} from './zero-dispatcher.ts';\n\nconst startupMessageEnv = 'ZERO_ENABLE_STARTUP_MESSAGE';\n\nfunction printStartupMessage(env: NodeJS.ProcessEnv) {\n  if (env[startupMessageEnv] !== '1') {\n    return;\n  }\n\n  colorConsole.log(\n    `\\nBTW, ${styleText(['bold', 'cyan'], 'Cloud Zero')} ` +\n      'is now available - professional Zero hosting from the team that built it.\\n' +\n      `Get started now: ${styleText(['blue', 'underline'], 'https://zero.rocicorp.dev/cloud')}\\n\\n` +\n      styleText('dim', `Disable this message with ${startupMessageEnv}=0`) +\n      '\\n',\n  );\n}\n\n/**\n * Top-level `runner` entry point to the zero-cache. This layer is responsible for:\n * * runtime-based config normalization\n * * lazy startup\n * * serving /statsz\n * * auto-reset restarts (TODO)\n */\nexport async function runWorker(\n  parent: Worker | null,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  // Note: Deprecation warnings are only emitted at this top-level parse;\n  //       they are suppressed when parsed in subprocesses.\n  const cfg = getZeroConfig({env, emitDeprecationWarnings: true});\n  const lc = createLogContext(cfg, 'runner');\n\n  const defaultTaskID = await getTaskID(lc);\n  const config = normalizeZeroConfig(lc, cfg, env, defaultTaskID);\n  const processes = new ProcessManager(lc, parent ?? process);\n\n  const {port, keepaliveTimeoutMs, lazyStartup, selfDrainDelayMs} = config;\n  const serverVersion = getServerVersion(config);\n  lc.info?.(`starting server${!serverVersion ? '' : `@${serverVersion}`} `, {\n    protocolVersion: PROTOCOL_VERSION,\n    taskID: config.taskID,\n    app: config.app,\n    shard: config.shard,\n    port: config.port,\n  });\n\n  const dispatcher = new ZeroDispatcher(\n    config,\n    lc,\n    {port, keepaliveTimeoutMs},\n    startZeroCache,\n    () => printStartupMessage(env),\n  );\n  const selfDrain = new SelfDrain(lc, parent ?? process, {\n    delayMs: selfDrainDelayMs,\n    onRequested: () => dispatcher.failHealthChecks(),\n  });\n\n  let zeroCache: Resolver<Worker> | undefined;\n  function startZeroCache(): Promise<Worker> {\n    if (zeroCache === undefined) {\n      const startMs = performance.now();\n      lc.info?.('starting zero-cache');\n\n      const r = (zeroCache = resolver<Worker>());\n      const w = childWorker(MAIN_URL, env)\n        .once('message', () => {\n          r.resolve(w);\n          lc.info?.(`zero-cache ready (${performance.now() - startMs} ms)`);\n        })\n        .once('error', r.reject);\n      w.onMessageType<DrainRequestedMessage>('drainRequested', () =>\n        selfDrain.request(),\n      );\n\n      processes.addWorker(w, 'user-facing', 'zero-cache');\n    }\n    return zeroCache.promise;\n  }\n\n  // Eagerly start the zero-cache if it was not configured with --lazy-startup.\n  if (!lazyStartup) {\n    void startZeroCache();\n  }\n\n  await processes.allWorkersReady();\n  parent?.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(lc, parent ?? process, dispatcher);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'main');\n  } finally {\n    await processes.shutdown();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAuBA,IAAM,oBAAoB;AAE1B,SAAS,oBAAoB,KAAwB;CACnD,IAAI,IAAI,uBAAuB,KAC7B;CAGF,aAAa,IACX,UAAU,UAAU,CAAC,QAAQ,MAAM,GAAG,YAAY,EAAE;mBAE9B,UAAU,CAAC,QAAQ,WAAW,GAAG,iCAAiC,EAAE,QACxF,UAAU,OAAO,6BAA6B,kBAAkB,GAAG,IACnE,IACJ;AACF;;;;;;;;AASA,eAAsB,UACpB,QACA,KACe;CAGf,MAAM,MAAM,cAAc;EAAC;EAAK,yBAAyB;CAAI,CAAC;CAC9D,MAAM,KAAK,iBAAiB,KAAK,QAAQ;CAEzC,MAAM,gBAAgB,MAAM,UAAU,EAAE;CACxC,MAAM,SAAS,oBAAoB,IAAI,KAAK,KAAK,aAAa;CAC9D,MAAM,YAAY,IAAI,eAAe,IAAI,UAAU,OAAO;CAE1D,MAAM,EAAC,MAAM,oBAAoB,aAAa,qBAAoB;CAClE,MAAM,gBAAgB,iBAAiB,MAAM;CAC7C,GAAG,OAAO,kBAAkB,CAAC,gBAAgB,KAAK,IAAI,gBAAgB,IAAI;EACxE,iBAAA;EACA,QAAQ,OAAO;EACf,KAAK,OAAO;EACZ,OAAO,OAAO;EACd,MAAM,OAAO;CACf,CAAC;CAED,MAAM,aAAa,IAAI,eACrB,QACA,IACA;EAAC;EAAM;CAAkB,GACzB,sBACM,oBAAoB,GAAG,CAC/B;CACA,MAAM,YAAY,IAAI,UAAU,IAAI,UAAU,SAAS;EACrD,SAAS;EACT,mBAAmB,WAAW,iBAAiB;CACjD,CAAC;CAED,IAAI;CACJ,SAAS,iBAAkC;EACzC,IAAI,cAAc,KAAA,GAAW;GAC3B,MAAM,UAAU,YAAY,IAAI;GAChC,GAAG,OAAO,qBAAqB;GAE/B,MAAM,IAAK,YAAY,SAAiB;GACxC,MAAM,IAAI,YAAY,UAAU,GAAG,CAAC,CACjC,KAAK,iBAAiB;IACrB,EAAE,QAAQ,CAAC;IACX,GAAG,OAAO,qBAAqB,YAAY,IAAI,IAAI,QAAQ,KAAK;GAClE,CAAC,CAAC,CACD,KAAK,SAAS,EAAE,MAAM;GACzB,EAAE,cAAqC,wBACrC,UAAU,QAAQ,CACpB;GAEA,UAAU,UAAU,GAAG,eAAe,YAAY;EACpD;EACA,OAAO,UAAU;CACnB;CAGA,IAAI,CAAC,aACH,eAAoB;CAGtB,MAAM,UAAU,gBAAgB;CAChC,QAAQ,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAErC,IAAI;EACF,MAAM,eAAe,IAAI,UAAU,SAAS,UAAU;CACxD,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,MAAM;CACvC,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF"}
+{"version":3,"file":"run-worker.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"sourcesContent":["import '../../../../shared/src/dotenv.ts';\n\nimport {styleText} from 'node:util';\nimport {colorConsole} from '../../../../shared/src/logging.ts';\nimport {resolver, type Resolver} from '../../../../shared/src/resolver.ts';\nimport {PROTOCOL_VERSION} from '../../../../zero-protocol/src/protocol-version.ts';\nimport {normalizeZeroConfig} from '../../config/normalize.ts';\nimport {getServerVersion, getZeroConfig} from '../../config/zero-config.ts';\nimport {\n  ProcessManager,\n  runUntilKilled,\n  SelfDrain,\n} from '../../services/life-cycle.ts';\nimport {\n  childWorker,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../../types/processes.ts';\nimport {createLogContext} from '../logging.ts';\nimport {MAIN_URL} from '../worker-urls.ts';\nimport {getTaskID} from './runtime.ts';\nimport {ZeroDispatcher} from './zero-dispatcher.ts';\n\nconst startupMessageEnv = 'ZERO_ENABLE_STARTUP_MESSAGE';\n\nfunction printStartupMessage(env: NodeJS.ProcessEnv) {\n  if (env[startupMessageEnv] !== '1') {\n    return;\n  }\n\n  colorConsole.log(\n    `\\nBTW, ${styleText(['bold', 'cyan'], 'Cloud Zero')} ` +\n      'is now available - professional Zero hosting from the team that built it.\\n' +\n      `Get started now: ${styleText(['blue', 'underline'], 'https://zero.rocicorp.dev/cloud')}\\n\\n` +\n      styleText('dim', `Disable this message with ${startupMessageEnv}=0`) +\n      '\\n',\n  );\n}\n\n/**\n * Top-level `runner` entry point to the zero-cache. This layer is responsible for:\n * * runtime-based config normalization\n * * lazy startup\n * * serving /statsz\n * * auto-reset restarts (TODO)\n */\nexport async function runWorker(\n  parent: Worker | null,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  // Note: Deprecation warnings are only emitted at this top-level parse;\n  //       they are suppressed when parsed in subprocesses.\n  const cfg = getZeroConfig({env, emitDeprecationWarnings: true});\n  const lc = createLogContext(cfg, 'runner');\n\n  const defaultTaskID = await getTaskID(lc);\n  const config = normalizeZeroConfig(lc, cfg, env, defaultTaskID);\n  const processes = new ProcessManager(lc, parent ?? process);\n\n  const {port, keepaliveTimeoutMs, lazyStartup, selfDrainDelayMs} = config;\n  const serverVersion = getServerVersion(config);\n  lc.info?.(`starting server${!serverVersion ? '' : `@${serverVersion}`} `, {\n    protocolVersion: PROTOCOL_VERSION,\n    taskID: config.taskID,\n    app: config.app,\n    shard: config.shard,\n    port: config.port,\n  });\n\n  const dispatcher = new ZeroDispatcher(\n    config,\n    lc,\n    {port, keepaliveTimeoutMs},\n    startZeroCache,\n    () => printStartupMessage(env),\n  );\n  const selfDrain = new SelfDrain(lc, parent ?? process, {\n    delayMs: selfDrainDelayMs,\n    onRequested: () => dispatcher.failHealthChecks(),\n  });\n\n  let zeroCache: Resolver<Worker> | undefined;\n  function startZeroCache(): Promise<Worker> {\n    if (zeroCache === undefined) {\n      const startMs = performance.now();\n      lc.info?.('starting zero-cache');\n\n      const r = (zeroCache = resolver<Worker>());\n      const w = childWorker(MAIN_URL, env)\n        .once('message', () => {\n          r.resolve(w);\n          lc.info?.(`zero-cache ready (${performance.now() - startMs} ms)`);\n        })\n        .once('error', r.reject);\n      w.onMessageType<DrainRequestedMessage>('drainRequested', () =>\n        selfDrain.request(),\n      );\n\n      processes.addWorker(w, 'user-facing', 'zero-cache');\n    }\n    return zeroCache.promise;\n  }\n\n  // Eagerly start the zero-cache if it was not configured with --lazy-startup.\n  if (!lazyStartup) {\n    void startZeroCache();\n  }\n\n  await processes.allWorkersReady();\n  parent?.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(lc, parent ?? process, dispatcher);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'main');\n  } finally {\n    await processes.shutdown();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAuBA,IAAM,oBAAoB;AAE1B,SAAS,oBAAoB,KAAwB;CACnD,IAAI,IAAI,uBAAuB,KAC7B;CAGF,aAAa,IACX,UAAU,UAAU,CAAC,QAAQ,MAAM,GAAG,YAAY,EAAE;mBAE9B,UAAU,CAAC,QAAQ,WAAW,GAAG,iCAAiC,EAAE,QACxF,UAAU,OAAO,6BAA6B,kBAAkB,GAAG,IACnE,IACJ;AACF;;;;;;;;AASA,eAAsB,UACpB,QACA,KACe;CAGf,MAAM,MAAM,cAAc;EAAC;EAAK,yBAAyB;CAAI,CAAC;CAC9D,MAAM,KAAK,iBAAiB,KAAK,QAAQ;CAEzC,MAAM,gBAAgB,MAAM,UAAU,EAAE;CACxC,MAAM,SAAS,oBAAoB,IAAI,KAAK,KAAK,aAAa;CAC9D,MAAM,YAAY,IAAI,eAAe,IAAI,UAAU,OAAO;CAE1D,MAAM,EAAC,MAAM,oBAAoB,aAAa,qBAAoB;CAClE,MAAM,gBAAgB,iBAAiB,MAAM;CAC7C,GAAG,OAAO,kBAAkB,CAAC,gBAAgB,KAAK,IAAI,gBAAgB,IAAI;EACxE,iBAAA;EACA,QAAQ,OAAO;EACf,KAAK,OAAO;EACZ,OAAO,OAAO;EACd,MAAM,OAAO;CACf,CAAC;CAED,MAAM,aAAa,IAAI,eACrB,QACA,IACA;EAAC;EAAM;CAAkB,GACzB,sBACM,oBAAoB,GAAG,CAC/B;CACA,MAAM,YAAY,IAAI,UAAU,IAAI,UAAU,SAAS;EACrD,SAAS;EACT,mBAAmB,WAAW,iBAAiB;CACjD,CAAC;CAED,IAAI;CACJ,SAAS,iBAAkC;EACzC,IAAI,cAAc,KAAA,GAAW;GAC3B,MAAM,UAAU,YAAY,IAAI;GAChC,GAAG,OAAO,qBAAqB;GAE/B,MAAM,IAAK,YAAY,SAAiB;GACxC,MAAM,IAAI,YAAY,UAAU,GAAG,CAAC,CACjC,KAAK,iBAAiB;IACrB,EAAE,QAAQ,CAAC;IACX,GAAG,OAAO,qBAAqB,YAAY,IAAI,IAAI,QAAQ,KAAK;GAClE,CAAC,CAAC,CACD,KAAK,SAAS,EAAE,MAAM;GACzB,EAAE,cAAqC,wBACrC,UAAU,QAAQ,CACpB;GAEA,UAAU,UAAU,GAAG,eAAe,YAAY;EACpD;EACA,OAAO,UAAU;CACnB;CAGA,IAAI,CAAC,aACH,eAAoB;CAGtB,MAAM,UAAU,gBAAgB;CAChC,QAAQ,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAErC,IAAI;EACF,MAAM,eAAe,IAAI,UAAU,SAAS,UAAU;CACxD,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,MAAM;CACvC,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.js

@@ -1,4 +1,5 @@
 import { assert } from "../../../../../shared/src/asserts.js";
+import { resolver } from "../../../../../shared/src/resolver.js";
 import { AbortError } from "../../../../../shared/src/abort-error.js";
 import { must } from "../../../../../shared/src/must.js";
 import { promiseOrAbort } from "../../../../../shared/src/promise-race.js";
@@ -8,7 +9,6 @@ import "../../../types/lite.js";
 import { stateVersionFromString, stateVersionToString } from "../../../types/state-version.js";
 import { Queue } from "../../../../../shared/src/queue.js";
 import { CustomKeyMap } from "../../../../../shared/src/custom-key-map.js";
-import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-source/common/backfill-manager.ts
 function tableKey({ schema, name }) {
 	return `${schema}.${name}`;

package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-manager.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseOrAbort} from '../../../../../shared/src/promise-race.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\n/**\n * A {@link MessageBackfill} or {@link BackfillCompleted} paired with the\n * approximate number of upstream bytes it represents. The BackfillManager uses\n * `byteSize` to bound the size of each replica transaction, committing and\n * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.\n */\nexport type BackfillMessage = {\n  message: MessageBackfill | BackfillCompleted;\n  byteSize: number;\n};\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<BackfillMessage>;\n\n/**\n * Commit (and reopen) the current backfill transaction once this many bytes of\n * backfill data have accumulated in it. Bounds the size of a single replica\n * COMMIT/checkpoint so it can't monopolize the replica.\n */\nconst COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;\n\n/**\n * How many backfill messages the prefetch producer may read ahead of the\n * reservation-holding consumer loop. This bounds the in-memory buffer (~this\n * many {@link BackfillMessage}s, each up to the stream's flush threshold, e.g.\n * 64 KiB) while giving the consumer a supply of ready messages so it never\n * has to wait on the upstream COPY while holding the change-stream reservation.\n */\nconst BACKFILL_PREFETCH_CREDITS = 16;\n\n/**\n * While the backfill holds an open transaction but its prefetch queue is empty\n * (i.e. the upstream is momentarily behind), the consumer polls for the next\n * message at this interval so that a replication transaction that begins\n * waiting mid-fetch can preempt the backfill within this bound, rather than\n * being blocked for the entire (possibly slow) upstream read.\n *\n * If stats indicate that these stalls are highly correlative with replication\n * lag (indicating that replication changes are waiting on these polls), we\n * can address this by adding an API to the change stream multiplexer that\n * would allow backfill to relinquish an in-progress commit on demand (i.e.\n * only if replication asks for a reservation).\n */\nconst BACKFILL_WAITER_POLL_MS = 10;\n\n/** Interval at which in-progress backfill timing stats are logged. */\nconst BACKFILL_STATS_LOG_INTERVAL_MS = 10_000;\n\n/** Sentinel enqueued by the prefetch producer when the backfill stream ends. */\nconst DONE = Symbol('backfill-done');\n/** Sentinel returned by a polled dequeue that timed out (see {@link BACKFILL_WAITER_POLL_MS}). */\nconst POLL = Symbol('backfill-poll');\n\ntype Pull = BackfillMessage | typeof DONE | typeof POLL;\n\n/**\n * Accumulated timing attribution for a single backfill run, logged\n * periodically to reveal where the backfill spends its time (and, in\n * particular, where the change-stream reservation is held). `produceMs`\n * accrues on the prefetch producer task and thus overlaps with the consumer's\n * `pushMs`; the others accrue on the reservation-holding consumer loop.\n */\ntype BackfillTimingStats = {\n  start: number;\n  lastLogged: number;\n  messages: number;\n  bytes: number;\n  commits: number;\n  stalls: number;\n  produceMs: number;\n  pushMs: number;\n  reserveMs: number;\n  stallMs: number;\n  waitMs: number;\n};\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  minWatermark: string;\n  /**\n   * Aborted (with the reason, if any) when this backfill is stopped,\n   * e.g. canceled.\n   */\n  stopped: AbortController;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  readonly #commitThresholdBytes: number;\n\n  /**\n   * Aborted when the change stream is canceled. No further backfills are run.\n   *\n   * TODO: Thread this into the backfillStream so that upstream I/O can be\n   * aborted.\n   */\n  readonly #controller = new AbortController();\n\n  get #canceled() {\n    return this.#controller.signal.aborted;\n  }\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n    commitThresholdBytes = COMMIT_THRESHOLD_BYTES,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n    this.#commitThresholdBytes = commitThresholdBytes;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#canceled &&\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {\n        request,\n        minWatermark: '',\n        stopped: new AbortController(),\n      };\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    if (this.#canceled) {\n      this.#lc.debug?.(`not retrying backfill: change stream canceled`, e);\n      return;\n    }\n    this.#lc.warn?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n    let uncommittedBytes = 0;\n\n    const stats: BackfillTimingStats = {\n      start: performance.now(),\n      lastLogged: performance.now(),\n      messages: 0,\n      bytes: 0,\n      commits: 0,\n      stalls: 0,\n      produceMs: 0,\n      pushMs: 0,\n      reserveMs: 0,\n      stallMs: 0,\n      waitMs: 0,\n    };\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.stopped.signal.aborted ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (!state.stopped.signal.aborted) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      let tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      if (tx < msg.watermark) {\n        // At this point it must be the case that the #changeStreamReached()\n        // the backfill watermark (possibly via a status message, which does\n        // not advance the reservation watermark). Given that guarantee,\n        // ensure that the version of the backfill transaction is at least up\n        // to the backfill watermark, so that the database state version is\n        // never earlier than the version of any backfilled rows.\n        tx = msg.watermark;\n      }\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n        stats.commits++;\n      }\n      backfillTx = null;\n      uncommittedBytes = 0;\n    };\n\n    // === Prefetch producer ===\n    //\n    // The backfill stream (an upstream Postgres COPY) is drained on a separate\n    // task into a bounded queue, so that the reservation-holding consumer loop\n    // below never blocks on the upstream read while it holds the change-stream\n    // reservation. `credits` bounds how far the producer runs ahead; the\n    // consumer returns a credit for every message it takes.\n    const ready = new Queue<Pull>();\n    const credits = new Queue<1>();\n    for (let i = 0; i < BACKFILL_PREFETCH_CREDITS; i++) {\n      credits.enqueue(1);\n    }\n    let stopProducer = false;\n\n    // Wakes a consumer blocked awaiting the next prefetched message so a\n    // canceled backfill unwinds promptly.\n    const onCancel = () =>\n      ready.enqueueRejection(new AbortError('backfill canceled'));\n    if (!this.#canceled) {\n      this.#controller.signal.addEventListener('abort', onCancel);\n    }\n\n    const producer = (async () => {\n      try {\n        let t0 = performance.now();\n        for await (const value of this.#backfillStreamer(state.request)) {\n          stats.produceMs += performance.now() - t0;\n          await credits.dequeue(); // wait for buffer space (flow control)\n          if (stopProducer || this.#canceled) {\n            ready.enqueueRejection(new AbortError('backfill canceled'));\n            return;\n          }\n          ready.enqueue(value);\n          t0 = performance.now();\n        }\n        ready.enqueue(DONE);\n      } catch (e) {\n        ready.enqueueRejection(e);\n      }\n    })();\n    // The producer routes errors through `ready` and never rejects; guard\n    // anyway so that leaving it un-awaited on the cancellation path (below)\n    // cannot surface as an unhandled rejection.\n    void producer.catch(() => {});\n\n    // === Consumer ===\n\n    // Fetches the next item from the prefetch queue. If a transaction is open\n    // but the queue is empty (the producer/upstream is momentarily behind),\n    // polls so that a replication transaction that begins waiting during the\n    // upstream read can preempt the backfill: rather than hold the reservation\n    // for the whole read, commit and release, letting replication proceed.\n    const nextItem = async (): Promise<BackfillMessage | typeof DONE> => {\n      // Fast path: a message is already buffered, or there's no reservation to\n      // protect. Take it without altering transaction boundaries. `POLL` is\n      // never enqueued (only used as a dequeue timeout value), so the result is\n      // always a message or DONE.\n      if (backfillTx === null || ready.size() > 0) {\n        const n = ready.dequeue();\n        const v = n instanceof Promise ? await n : n;\n        return v as BackfillMessage | typeof DONE;\n      }\n      // Slow path: holding a reservation with an empty queue.\n      for (;;) {\n        if (backfillTx !== null && changeStream.waiterDelay() > 0) {\n          commitTx(); // yield the stream to the waiting replication txn\n        }\n        const n =\n          backfillTx !== null\n            ? ready.dequeue(POLL, BACKFILL_WAITER_POLL_MS)\n            : ready.dequeue(); // released: just wait for the next message\n        const t0 = performance.now();\n        const v = n instanceof Promise ? await n : n;\n        if (v === POLL) {\n          stats.stallMs += performance.now() - t0; // timed out; re-check waiter\n          continue;\n        }\n        if (n instanceof Promise) {\n          stats.stallMs += performance.now() - t0;\n          stats.stalls++;\n        }\n        return v;\n      }\n    };\n\n    try {\n      for (;;) {\n        const item = await nextItem();\n        if (item === DONE) {\n          break;\n        }\n        // Allow the producer to fetch the next message concurrently with the\n        // processing (and downstream push) of this one.\n        credits.enqueue(1);\n\n        if (this.#canceled) {\n          // Returning finalizes the backfill stream (and the upstream resources\n          // it holds) via the producer's `finally`. The reservation, if held,\n          // does not need to be released since the change stream is gone.\n          lc.info?.(`backfill stream canceled: change stream canceled`);\n          return;\n        }\n        const {message: msg, byteSize} = item;\n\n        // Before sending any backfill message, the main replication stream\n        // must reach the backfill (snapshot) watermark. Otherwise, snapshot\n        // rows can constitute \"future\" state (e.g. phantom rows at keys that\n        // the stream has yet to move or insert) that the subsequent\n        // replication changes conflict with (e.g. on unique indexes).\n        const waitForChangeStream = this.#changeStreamReached(\n          lc,\n          msg.watermark,\n        );\n\n        // Commit (and later reopen) the transaction if the main stream is\n        // waiting on the reservation, if we must wait for the main stream, or\n        // if the size of current transaction has reached the commit threshold.\n        if (\n          backfillTx &&\n          (changeStream.waiterDelay() > 0 ||\n            waitForChangeStream ||\n            uncommittedBytes >= this.#commitThresholdBytes)\n        ) {\n          commitTx();\n        }\n\n        if (waitForChangeStream) {\n          const t0 = performance.now();\n          // Exit promptly (via the error path) if the backfill is canceled\n          // while waiting, releasing its upstream resources.\n          await promiseOrAbort(waitForChangeStream, state.stopped.signal);\n          const elapsed = performance.now() - t0;\n          stats.waitMs += elapsed;\n          lc.info?.(\n            `change stream reached ${msg.watermark} ` +\n              `after waiting ${Math.round(elapsed)} ms`,\n          );\n        }\n\n        if (\n          msg.tag === 'backfill' &&\n          msg.rowValues.length > 0 &&\n          msg.relation.rowKey.columns.length === 0\n        ) {\n          throw new MissingRowKeyError(state.request);\n        }\n\n        // Reserve the changeStreamer if not in a transaction.\n        if (backfillTx === null) {\n          const t0 = performance.now();\n          backfillTx = await beginTxFor(msg);\n          stats.reserveMs += performance.now() - t0;\n          if (backfillTx === null) {\n            lc.info?.(\n              `backfill stream canceled: ${state.stopped.signal.reason}`,\n              state.request,\n            );\n            this.#checkAndStartBackfill(); // start the next backfill if present\n            return; // this backfill is canceled\n          }\n        }\n\n        // `await` to allow the change streamer to exert back pressure\n        // on backfills.\n        const t0 = performance.now();\n        await changeStream.push(['data', msg]);\n        stats.pushMs += performance.now() - t0;\n        uncommittedBytes += byteSize;\n        stats.messages++;\n        stats.bytes += byteSize;\n\n        this.#maybeLogBackfillStats(lc, stats);\n      }\n\n      // Flush any final tx and release the stream.\n      if (backfillTx) {\n        commitTx();\n      }\n      lc.debug?.(`backfill stream exited`);\n    } finally {\n      stopProducer = true;\n      credits.enqueue(1);\n      // The producer can potentially be hung on upstream I/O.\n      // TODO: Thread AbortSignal logic into the backfillStreamer.\n      if (!this.#canceled) {\n        await producer;\n      }\n      this.#controller.signal.removeEventListener('abort', onCancel);\n      this.#logBackfillStats(lc, stats);\n    }\n  }\n\n  #maybeLogBackfillStats(lc: LogContext, stats: BackfillTimingStats) {\n    const now = performance.now();\n    if (now - stats.lastLogged >= BACKFILL_STATS_LOG_INTERVAL_MS) {\n      this.#logBackfillStats(lc, stats);\n      stats.lastLogged = now;\n    }\n  }\n\n  /**\n   * Logs where the backfill spent its time. `produceMs` (upstream read +\n   * parse/decode) accrues on the prefetch producer concurrently with the\n   * consumer's `pushMs` (downstream backpressure), so the two can overlap and\n   * their sum can exceed `elapsedMs`. `reserveMs` is time spent (re)acquiring\n   * the change-stream reservation, and `stallMs` is time the consumer waited on\n   * an empty prefetch queue (i.e. blocked on the upstream) — a large `stallMs`\n   * indicates an upstream-bound backfill. `waitMs` is time spent waiting\n   * (without the reservation) for the change stream to reach the backfill\n   * watermark, during which the upstream snapshot is held open.\n   */\n  #logBackfillStats(lc: LogContext, stats: BackfillTimingStats) {\n    const elapsed = performance.now() - stats.start;\n    lc.info?.(`backfill timing`, {\n      messages: stats.messages,\n      bytes: stats.bytes,\n      commits: stats.commits,\n      stalls: stats.stalls,\n      elapsedMs: Math.round(elapsed),\n      produceMs: Math.round(stats.produceMs),\n      pushMs: Math.round(stats.pushMs),\n      reserveMs: Math.round(stats.reserveMs),\n      stallMs: Math.round(stats.stallMs),\n      waitMs: Math.round(stats.waitMs),\n      throughputKBps:\n        elapsed > 0 ? Math.round(stats.bytes / 1024 / (elapsed / 1000)) : 0,\n    });\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.stopped.abort(reason);\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: Object.fromEntries(\n              Object.entries(backfill).map(([col, id]) => [col, {id}]),\n            ),\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: {id: backfill}},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: {id: backfill},\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#controller.abort('canceled');\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n    this.#backfillRetryTimer = undefined;\n\n    // Wake up a backfill that is waiting for the change stream to reach a\n    // watermark. The change stream never will, and the backfill must be\n    // allowed to unwind (and release its upstream resources) rather than\n    // remain suspended forever.\n    for (const {reached} of this.#awaitingStatusWatermarks.splice(0)) {\n      reached();\n    }\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"mappings":";;;;;;;;;;;;AA4BA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;;;;;;AAsBA,IAAM,yBAAyB;;;;;;;;AAS/B,IAAM,4BAA4B;;;;;;;;;;;;;;AAelC,IAAM,0BAA0B;;AAGhC,IAAM,iCAAiC;;AAGvC,IAAM,OAAO,OAAO,eAAe;;AAEnC,IAAM,OAAO,OAAO,eAAe;AAmCnC,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC;;;;;;;CAQA,cAAuB,IAAI,gBAAgB;CAE3C,IAAI,YAAY;EACd,OAAO,KAAK,YAAY,OAAO;CACjC;CAEA,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf,uBAAuB,wBACvB;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAK,kBAAkB;EACvB,KAAK,oBAAoB;EACzB,KAAK,cAAc;EACnB,KAAK,gBAAgB;EACrB,KAAK,gBAAgB;EACrB,KAAK,gBAAgB;EACrB,KAAK,wBAAwB;CAC/B;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAK,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAK,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAK,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAK,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAK,wBAAwB,MAAM,WAAW;GACjD,KAAK,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAK,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAK,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAK,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAK,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAK,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAK,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAK,aACN,CAAC,KAAK,uBACN,CAAC,KAAK,oBACN,KAAK,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAK,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IACZ;IACA,cAAc;IACd,SAAS,IAAI,gBAAgB;GAC/B;GACA,MAAM,KAAK,KAAK,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAK,mBAAmB;GACxB,KAAU,aAAa,IAAI,KAAK,CAAC,CAC9B,WAAW;IACV,KAAK,qBAAqB,mBAAmB,KAAK;IAClD,KAAK,gBAAgB,KAAK;GAC5B,CAAC,CAAC,CAGD,OAAM,MAAK;IACV,KAAK,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAK,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,IAAI,KAAK,WAAW;GAClB,KAAK,IAAI,QAAQ,iDAAiD,CAAC;GACnE;EACF;EACA,KAAK,IAAI,OACP,uCAAuC,KAAK,cAAc,MAC1D,CACF;EACA,KAAK,sBAAsB,iBAAiB;GAC1C,KAAK,sBAAsB,KAAA;GAC3B,KAAK,uBAAuB;EAC9B,GAAG,KAAK,aAAa;EAErB,KAAK,gBAAgB,KAAK,IAAI,KAAK,gBAAgB,GAAG,KAAK,aAAa;CAC1E;CAEA,MAAM,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAK;EAI1B,IAAI,aAA4B;EAChC,IAAI,mBAAmB;EAEvB,MAAM,QAA6B;GACjC,OAAO,YAAY,IAAI;GACvB,YAAY,YAAY,IAAI;GAC5B,UAAU;GACV,OAAO;GACP,SAAS;GACT,QAAQ;GACR,WAAW;GACX,QAAQ;GACR,WAAW;GACX,SAAS;GACT,QAAQ;EACV;;;;EAKA,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,QAAQ,OAAO,WACpB,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,CAAC,MAAM,QAAQ,OAAO,SAAS;KACjC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAK,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,KAAK,IAAI,WAOX,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAK;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;IAC/B,MAAM;GACR;GACA,aAAa;GACb,mBAAmB;EACrB;EASA,MAAM,QAAQ,IAAI,MAAY;EAC9B,MAAM,UAAU,IAAI,MAAS;EAC7B,KAAK,IAAI,IAAI,GAAG,IAAI,2BAA2B,KAC7C,QAAQ,QAAQ,CAAC;EAEnB,IAAI,eAAe;EAInB,MAAM,iBACJ,MAAM,iBAAiB,IAAI,WAAW,mBAAmB,CAAC;EAC5D,IAAI,CAAC,KAAK,WACR,KAAK,YAAY,OAAO,iBAAiB,SAAS,QAAQ;EAG5D,MAAM,YAAY,YAAY;GAC5B,IAAI;IACF,IAAI,KAAK,YAAY,IAAI;IACzB,WAAW,MAAM,SAAS,KAAK,kBAAkB,MAAM,OAAO,GAAG;KAC/D,MAAM,aAAa,YAAY,IAAI,IAAI;KACvC,MAAM,QAAQ,QAAQ;KACtB,IAAI,gBAAgB,KAAK,WAAW;MAClC,MAAM,iBAAiB,IAAI,WAAW,mBAAmB,CAAC;MAC1D;KACF;KACA,MAAM,QAAQ,KAAK;KACnB,KAAK,YAAY,IAAI;IACvB;IACA,MAAM,QAAQ,IAAI;GACpB,SAAS,GAAG;IACV,MAAM,iBAAiB,CAAC;GAC1B;EACF,EAAA,CAAG;EAIH,SAAc,YAAY,CAAC,CAAC;EAS5B,MAAM,WAAW,YAAoD;GAKnE,IAAI,eAAe,QAAQ,MAAM,KAAK,IAAI,GAAG;IAC3C,MAAM,IAAI,MAAM,QAAQ;IAExB,OADU,aAAa,UAAU,MAAM,IAAI;GAE7C;GAEA,SAAS;IACP,IAAI,eAAe,QAAQ,aAAa,YAAY,IAAI,GACtD,SAAS;IAEX,MAAM,IACJ,eAAe,OACX,MAAM,QAAQ,MAAM,uBAAuB,IAC3C,MAAM,QAAQ;IACpB,MAAM,KAAK,YAAY,IAAI;IAC3B,MAAM,IAAI,aAAa,UAAU,MAAM,IAAI;IAC3C,IAAI,MAAM,MAAM;KACd,MAAM,WAAW,YAAY,IAAI,IAAI;KACrC;IACF;IACA,IAAI,aAAa,SAAS;KACxB,MAAM,WAAW,YAAY,IAAI,IAAI;KACrC,MAAM;IACR;IACA,OAAO;GACT;EACF;EAEA,IAAI;GACF,SAAS;IACP,MAAM,OAAO,MAAM,SAAS;IAC5B,IAAI,SAAS,MACX;IAIF,QAAQ,QAAQ,CAAC;IAEjB,IAAI,KAAK,WAAW;KAIlB,GAAG,OAAO,kDAAkD;KAC5D;IACF;IACA,MAAM,EAAC,SAAS,KAAK,aAAY;IAOjC,MAAM,sBAAsB,KAAK,qBAC/B,IACA,IAAI,SACN;IAKA,IACE,eACC,aAAa,YAAY,IAAI,KAC5B,uBACA,oBAAoB,KAAK,wBAE3B,SAAS;IAGX,IAAI,qBAAqB;KACvB,MAAM,KAAK,YAAY,IAAI;KAG3B,MAAM,eAAe,qBAAqB,MAAM,QAAQ,MAAM;KAC9D,MAAM,UAAU,YAAY,IAAI,IAAI;KACpC,MAAM,UAAU;KAChB,GAAG,OACD,yBAAyB,IAAI,UAAU,iBACpB,KAAK,MAAM,OAAO,EAAE,IACzC;IACF;IAEA,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;IAI5C,IAAI,eAAe,MAAM;KACvB,MAAM,KAAK,YAAY,IAAI;KAC3B,aAAa,MAAM,WAAW,GAAG;KACjC,MAAM,aAAa,YAAY,IAAI,IAAI;KACvC,IAAI,eAAe,MAAM;MACvB,GAAG,OACD,6BAA6B,MAAM,QAAQ,OAAO,UAClD,MAAM,OACR;MACA,KAAK,uBAAuB;MAC5B;KACF;IACF;IAIA,MAAM,KAAK,YAAY,IAAI;IAC3B,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;IACrC,MAAM,UAAU,YAAY,IAAI,IAAI;IACpC,oBAAoB;IACpB,MAAM;IACN,MAAM,SAAS;IAEf,KAAK,uBAAuB,IAAI,KAAK;GACvC;GAGA,IAAI,YACF,SAAS;GAEX,GAAG,QAAQ,wBAAwB;EACrC,UAAU;GACR,eAAe;GACf,QAAQ,QAAQ,CAAC;GAGjB,IAAI,CAAC,KAAK,WACR,MAAM;GAER,KAAK,YAAY,OAAO,oBAAoB,SAAS,QAAQ;GAC7D,KAAK,kBAAkB,IAAI,KAAK;EAClC;CACF;CAEA,uBAAuB,IAAgB,OAA4B;EACjE,MAAM,MAAM,YAAY,IAAI;EAC5B,IAAI,MAAM,MAAM,cAAc,gCAAgC;GAC5D,KAAK,kBAAkB,IAAI,KAAK;GAChC,MAAM,aAAa;EACrB;CACF;;;;;;;;;;;;CAaA,kBAAkB,IAAgB,OAA4B;EAC5D,MAAM,UAAU,YAAY,IAAI,IAAI,MAAM;EAC1C,GAAG,OAAO,mBAAmB;GAC3B,UAAU,MAAM;GAChB,OAAO,MAAM;GACb,SAAS,MAAM;GACf,QAAQ,MAAM;GACd,WAAW,KAAK,MAAM,OAAO;GAC7B,WAAW,KAAK,MAAM,MAAM,SAAS;GACrC,QAAQ,KAAK,MAAM,MAAM,MAAM;GAC/B,WAAW,KAAK,MAAM,MAAM,SAAS;GACrC,SAAS,KAAK,MAAM,MAAM,OAAO;GACjC,QAAQ,KAAK,MAAM,MAAM,MAAM;GAC/B,gBACE,UAAU,IAAI,KAAK,MAAM,MAAM,QAAQ,QAAQ,UAAU,IAAK,IAAI;EACtE,CAAC;CACH;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAK;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAK;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,QAAQ,MAAM,MAAM;GAC7B,KAAK,mBAAmB;GACxB,UAAU,KAAK,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAK,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAK,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAK,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAK,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAK,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAK,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAK,sBAAsB,QAAQ,EAAE,CAAC;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAK,sBAAsB;GAC3B,KAAK,wBAAwB,QAAQ,EAAE;GAGvC,KAAK,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAK,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAK,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAK,oBAAoB,KAAK,GAChC,KAAK,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAK,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS,OAAO,YACd,OAAO,QAAQ,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,KAAK,EAAC,GAAE,CAAC,CAAC,CACzD;IACF,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAK,wBAAwB,KAAK,GAAG;KACrC,KAAK,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAK,oBAAoB,GAAG,GAC9B,KAAK,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAK,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAK,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAK,oBAAoB,EAAE,GAC7B,KAAK,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAK,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,EAAC,IAAI,SAAQ,EAAC;KACzC,CAAC;UAED,KAAK,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO,EAAC,IAAI,SAAQ;MAC9B;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAK,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAK,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAK,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,CAAC,CAAC,WAAW,GACpC,KAAK,wBAAwB,KAAK,KAAK;UAEvC,KAAK,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAK,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAK,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAK,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAK,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ,MAQzC;UAAA,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAK,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,CAAC,CAAC,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAK,wBAAwB,KAAK,QAAQ;SAE1C,KAAK,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAK,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAK,YAAY,MAAM,UAAU;EACjC,KAAK,qBAAqB,wBAAwB;EAClD,aAAa,KAAK,mBAAmB;EACrC,KAAK,sBAAsB,KAAA;EAM3B,KAAK,MAAM,EAAC,aAAY,KAAK,0BAA0B,OAAO,CAAC,GAC7D,QAAQ;CAEZ;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,CAAC,CAAC,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}
+{"version":3,"file":"backfill-manager.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseOrAbort} from '../../../../../shared/src/promise-race.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {resolver} from '../../../../../shared/src/resolver.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n  stateVersionFromString,\n  stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  ChangeStreamMessage,\n  Identifier,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n  Cancelable,\n  ChangeStreamMultiplexer,\n  Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n  return `${schema}.${name}`;\n}\n\n/**\n * A {@link MessageBackfill} or {@link BackfillCompleted} paired with the\n * approximate number of upstream bytes it represents. The BackfillManager uses\n * `byteSize` to bound the size of each replica transaction, committing and\n * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.\n */\nexport type BackfillMessage = {\n  message: MessageBackfill | BackfillCompleted;\n  byteSize: number;\n};\n\ntype BackfillStreamer = (\n  req: BackfillRequest,\n) => AsyncGenerator<BackfillMessage>;\n\n/**\n * Commit (and reopen) the current backfill transaction once this many bytes of\n * backfill data have accumulated in it. Bounds the size of a single replica\n * COMMIT/checkpoint so it can't monopolize the replica.\n */\nconst COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;\n\n/**\n * How many backfill messages the prefetch producer may read ahead of the\n * reservation-holding consumer loop. This bounds the in-memory buffer (~this\n * many {@link BackfillMessage}s, each up to the stream's flush threshold, e.g.\n * 64 KiB) while giving the consumer a supply of ready messages so it never\n * has to wait on the upstream COPY while holding the change-stream reservation.\n */\nconst BACKFILL_PREFETCH_CREDITS = 16;\n\n/**\n * While the backfill holds an open transaction but its prefetch queue is empty\n * (i.e. the upstream is momentarily behind), the consumer polls for the next\n * message at this interval so that a replication transaction that begins\n * waiting mid-fetch can preempt the backfill within this bound, rather than\n * being blocked for the entire (possibly slow) upstream read.\n *\n * If stats indicate that these stalls are highly correlative with replication\n * lag (indicating that replication changes are waiting on these polls), we\n * can address this by adding an API to the change stream multiplexer that\n * would allow backfill to relinquish an in-progress commit on demand (i.e.\n * only if replication asks for a reservation).\n */\nconst BACKFILL_WAITER_POLL_MS = 10;\n\n/** Interval at which in-progress backfill timing stats are logged. */\nconst BACKFILL_STATS_LOG_INTERVAL_MS = 10_000;\n\n/** Sentinel enqueued by the prefetch producer when the backfill stream ends. */\nconst DONE = Symbol('backfill-done');\n/** Sentinel returned by a polled dequeue that timed out (see {@link BACKFILL_WAITER_POLL_MS}). */\nconst POLL = Symbol('backfill-poll');\n\ntype Pull = BackfillMessage | typeof DONE | typeof POLL;\n\n/**\n * Accumulated timing attribution for a single backfill run, logged\n * periodically to reveal where the backfill spends its time (and, in\n * particular, where the change-stream reservation is held). `produceMs`\n * accrues on the prefetch producer task and thus overlaps with the consumer's\n * `pushMs`; the others accrue on the reservation-holding consumer loop.\n */\ntype BackfillTimingStats = {\n  start: number;\n  lastLogged: number;\n  messages: number;\n  bytes: number;\n  commits: number;\n  stalls: number;\n  produceMs: number;\n  pushMs: number;\n  reserveMs: number;\n  stallMs: number;\n  waitMs: number;\n};\n\ntype RunningBackfillState = {\n  request: BackfillRequest;\n  minWatermark: string;\n  /**\n   * Aborted (with the reason, if any) when this backfill is stopped,\n   * e.g. canceled.\n   */\n  stopped: AbortController;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n  watermark: string;\n  reached: () => void;\n};\n\n/**\n * The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n *\n * The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n *\n * The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n */\nexport class BackfillManager implements Cancelable, Listener {\n  readonly #lc: LogContext;\n\n  /**\n   * Tracks the metadata of required backfills based on schema changes\n   * and initial backfill requests.\n   */\n  readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n    tableKey,\n  );\n  readonly #changeStreamer: ChangeStreamMultiplexer;\n  readonly #backfillStreamer: BackfillStreamer;\n  readonly #jsonFormat: JSONFormat;\n\n  /**\n   * The current running backfill. The backfill request is always also in\n   * `#requiredBackfills` (technically, it can be a subset of what's in\n   * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n   * upon completion.\n   */\n  #runningBackfill: RunningBackfillState | null = null;\n\n  /** The last seen watermark in the change stream. */\n  #lastStatusWatermark: string | null = null;\n\n  readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n  /** The watermark of the current transaction in the change stream. */\n  #currentTxWatermark: string | null = null;\n\n  readonly #commitThresholdBytes: number;\n\n  /**\n   * Aborted when the change stream is canceled. No further backfills are run.\n   *\n   * TODO: Thread this into the backfillStream so that upstream I/O can be\n   * aborted.\n   */\n  readonly #controller = new AbortController();\n\n  get #canceled() {\n    return this.#controller.signal.aborted;\n  }\n\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamMultiplexer,\n    backfillStreamer: BackfillStreamer,\n    jsonFormat: JSONFormat = JSON_STRINGIFIED,\n    minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n    maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n    commitThresholdBytes = COMMIT_THRESHOLD_BYTES,\n  ) {\n    this.#lc = lc.withContext('component', 'backfill-manager');\n    this.#changeStreamer = changeStreamer;\n    this.#backfillStreamer = backfillStreamer;\n    this.#jsonFormat = jsonFormat;\n    this.#minBackoffMs = minBackoffMs;\n    this.#maxBackoffMs = maxBackoffMs;\n    this.#retryDelayMs = minBackoffMs;\n    this.#commitThresholdBytes = commitThresholdBytes;\n  }\n\n  run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n    this.#lc.info?.(\n      `starting backfill manager with ${initialRequests.length} initial requests`,\n      {requests: initialRequests},\n    );\n    this.#lastStatusWatermark = lastWatermark;\n    initialRequests.forEach(req =>\n      this.#setRequiredBackfill('initial-request', req),\n    );\n    this.#checkAndStartBackfill();\n  }\n\n  #setLastStatusWatermark({watermark}: {watermark: string}) {\n    // Only allow the watermark to move forward. This prevents a backfill\n    // transaction (whose watermark is unrelated to change-stream state)\n    // from moving the watermark backwards.\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      this.#lastStatusWatermark = watermark;\n      for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n        const awaiting = this.#awaitingStatusWatermarks[i];\n        if (watermark >= awaiting.watermark) {\n          awaiting.reached();\n          this.#awaitingStatusWatermarks.splice(i, 1);\n        }\n      }\n    }\n  }\n\n  #changeStreamReached(\n    lc: LogContext,\n    watermark: string,\n  ): Promise<void> | null {\n    if ((this.#lastStatusWatermark ?? '') < watermark) {\n      const {promise, resolve: reached} = resolver();\n      this.#awaitingStatusWatermarks.push({watermark, reached});\n      lc.info?.(\n        `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n      );\n      return promise;\n    }\n    return null;\n  }\n\n  readonly #minBackoffMs: number;\n  readonly #maxBackoffMs: number;\n  #retryDelayMs: number;\n  #backfillRetryTimer: NodeJS.Timeout | undefined;\n\n  #checkAndStartBackfill() {\n    if (\n      !this.#canceled &&\n      !this.#backfillRetryTimer &&\n      !this.#runningBackfill &&\n      this.#requiredBackfills.size\n    ) {\n      // Pick a random backfill to avoid head-of-line blocking by a\n      // problematic backfill (e.g. awaiting a primary key). This is\n      // simpler that adding logic to classify (and declassify)\n      // problematic backfills.\n      const candidates = [...this.#requiredBackfills.values()];\n      const request = candidates[randInt(0, candidates.length - 1)];\n      const state = {\n        request,\n        minWatermark: '',\n        stopped: new AbortController(),\n      };\n      const lc = this.#lc.withContext('table', request.table.name);\n\n      this.#runningBackfill = state;\n      void this.#runBackfill(lc, state)\n        .then(() => {\n          this.#stopRunningBackfill('backfill exited', state);\n          this.#retryDelayMs = this.#minBackoffMs; // reset on success\n        })\n        // For unexpected errors (e.g. upstream replication slot\n        // unavailability), retry with exponential backoff.\n        .catch(e => {\n          this.#stopRunningBackfill(String(e), state);\n          this.#retryBackfillWithBackoff(e);\n        });\n    }\n  }\n\n  #retryBackfillWithBackoff(e: unknown) {\n    if (this.#canceled) {\n      this.#lc.debug?.(`not retrying backfill: change stream canceled`, e);\n      return;\n    }\n    this.#lc.warn?.(\n      `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n      e,\n    );\n    this.#backfillRetryTimer = setTimeout(() => {\n      this.#backfillRetryTimer = undefined;\n      this.#checkAndStartBackfill();\n    }, this.#retryDelayMs);\n\n    this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);\n  }\n\n  async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n    const changeStream = this.#changeStreamer; // Purely for readability\n\n    // backfillTx is set if and only if a changeStreamer reservation has been\n    // acquired and the backfill stream is inside a transaction.\n    let backfillTx: string | null = null;\n    let uncommittedBytes = 0;\n\n    const stats: BackfillTimingStats = {\n      start: performance.now(),\n      lastLogged: performance.now(),\n      messages: 0,\n      bytes: 0,\n      commits: 0,\n      stalls: 0,\n      produceMs: 0,\n      pushMs: 0,\n      reserveMs: 0,\n      stallMs: 0,\n      waitMs: 0,\n    };\n\n    /**\n     * @returns the new tx watermark, or null if backfill was cancelled\n     */\n    const beginTxFor = async (\n      msg: MessageBackfill | BackfillCompleted,\n    ): Promise<string | null> => {\n      assert(backfillTx === null, 'Expected no active backfill transaction');\n      const lastWatermark = await changeStream.reserve('backfill');\n\n      // After obtaining the changeStream reservation, check if the stream\n      // had changes that resulted in invalidating / canceling this backfill.\n      if (\n        state.stopped.signal.aborted ||\n        (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n      ) {\n        if (!state.stopped.signal.aborted) {\n          assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n          this.#stopRunningBackfill(\n            `row key change at ${state.minWatermark} ` +\n              `postdates backfill watermark at ${msg.watermark}`,\n            state,\n          );\n        }\n        changeStream.release(lastWatermark);\n        return null;\n      }\n\n      const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n      let tx = stateVersionToString({\n        major,\n        minor: BigInt(minor) + 1n,\n      });\n\n      if (tx < msg.watermark) {\n        // At this point it must be the case that the #changeStreamReached()\n        // the backfill watermark (possibly via a status message, which does\n        // not advance the reservation watermark). Given that guarantee,\n        // ensure that the version of the backfill transaction is at least up\n        // to the backfill watermark, so that the database state version is\n        // never earlier than the version of any backfilled rows.\n        tx = msg.watermark;\n      }\n\n      void changeStream.push([\n        'begin',\n        {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n        {commitWatermark: tx},\n      ]);\n      return (backfillTx = tx);\n    };\n\n    const commitTx = () => {\n      if (backfillTx) {\n        void changeStream.push([\n          'commit',\n          {tag: 'commit'},\n          {watermark: backfillTx},\n        ]);\n        changeStream.release(backfillTx);\n        stats.commits++;\n      }\n      backfillTx = null;\n      uncommittedBytes = 0;\n    };\n\n    // === Prefetch producer ===\n    //\n    // The backfill stream (an upstream Postgres COPY) is drained on a separate\n    // task into a bounded queue, so that the reservation-holding consumer loop\n    // below never blocks on the upstream read while it holds the change-stream\n    // reservation. `credits` bounds how far the producer runs ahead; the\n    // consumer returns a credit for every message it takes.\n    const ready = new Queue<Pull>();\n    const credits = new Queue<1>();\n    for (let i = 0; i < BACKFILL_PREFETCH_CREDITS; i++) {\n      credits.enqueue(1);\n    }\n    let stopProducer = false;\n\n    // Wakes a consumer blocked awaiting the next prefetched message so a\n    // canceled backfill unwinds promptly.\n    const onCancel = () =>\n      ready.enqueueRejection(new AbortError('backfill canceled'));\n    if (!this.#canceled) {\n      this.#controller.signal.addEventListener('abort', onCancel);\n    }\n\n    const producer = (async () => {\n      try {\n        let t0 = performance.now();\n        for await (const value of this.#backfillStreamer(state.request)) {\n          stats.produceMs += performance.now() - t0;\n          await credits.dequeue(); // wait for buffer space (flow control)\n          if (stopProducer || this.#canceled) {\n            ready.enqueueRejection(new AbortError('backfill canceled'));\n            return;\n          }\n          ready.enqueue(value);\n          t0 = performance.now();\n        }\n        ready.enqueue(DONE);\n      } catch (e) {\n        ready.enqueueRejection(e);\n      }\n    })();\n    // The producer routes errors through `ready` and never rejects; guard\n    // anyway so that leaving it un-awaited on the cancellation path (below)\n    // cannot surface as an unhandled rejection.\n    void producer.catch(() => {});\n\n    // === Consumer ===\n\n    // Fetches the next item from the prefetch queue. If a transaction is open\n    // but the queue is empty (the producer/upstream is momentarily behind),\n    // polls so that a replication transaction that begins waiting during the\n    // upstream read can preempt the backfill: rather than hold the reservation\n    // for the whole read, commit and release, letting replication proceed.\n    const nextItem = async (): Promise<BackfillMessage | typeof DONE> => {\n      // Fast path: a message is already buffered, or there's no reservation to\n      // protect. Take it without altering transaction boundaries. `POLL` is\n      // never enqueued (only used as a dequeue timeout value), so the result is\n      // always a message or DONE.\n      if (backfillTx === null || ready.size() > 0) {\n        const n = ready.dequeue();\n        const v = n instanceof Promise ? await n : n;\n        return v as BackfillMessage | typeof DONE;\n      }\n      // Slow path: holding a reservation with an empty queue.\n      for (;;) {\n        if (backfillTx !== null && changeStream.waiterDelay() > 0) {\n          commitTx(); // yield the stream to the waiting replication txn\n        }\n        const n =\n          backfillTx !== null\n            ? ready.dequeue(POLL, BACKFILL_WAITER_POLL_MS)\n            : ready.dequeue(); // released: just wait for the next message\n        const t0 = performance.now();\n        const v = n instanceof Promise ? await n : n;\n        if (v === POLL) {\n          stats.stallMs += performance.now() - t0; // timed out; re-check waiter\n          continue;\n        }\n        if (n instanceof Promise) {\n          stats.stallMs += performance.now() - t0;\n          stats.stalls++;\n        }\n        return v;\n      }\n    };\n\n    try {\n      for (;;) {\n        const item = await nextItem();\n        if (item === DONE) {\n          break;\n        }\n        // Allow the producer to fetch the next message concurrently with the\n        // processing (and downstream push) of this one.\n        credits.enqueue(1);\n\n        if (this.#canceled) {\n          // Returning finalizes the backfill stream (and the upstream resources\n          // it holds) via the producer's `finally`. The reservation, if held,\n          // does not need to be released since the change stream is gone.\n          lc.info?.(`backfill stream canceled: change stream canceled`);\n          return;\n        }\n        const {message: msg, byteSize} = item;\n\n        // Before sending any backfill message, the main replication stream\n        // must reach the backfill (snapshot) watermark. Otherwise, snapshot\n        // rows can constitute \"future\" state (e.g. phantom rows at keys that\n        // the stream has yet to move or insert) that the subsequent\n        // replication changes conflict with (e.g. on unique indexes).\n        const waitForChangeStream = this.#changeStreamReached(\n          lc,\n          msg.watermark,\n        );\n\n        // Commit (and later reopen) the transaction if the main stream is\n        // waiting on the reservation, if we must wait for the main stream, or\n        // if the size of current transaction has reached the commit threshold.\n        if (\n          backfillTx &&\n          (changeStream.waiterDelay() > 0 ||\n            waitForChangeStream ||\n            uncommittedBytes >= this.#commitThresholdBytes)\n        ) {\n          commitTx();\n        }\n\n        if (waitForChangeStream) {\n          const t0 = performance.now();\n          // Exit promptly (via the error path) if the backfill is canceled\n          // while waiting, releasing its upstream resources.\n          await promiseOrAbort(waitForChangeStream, state.stopped.signal);\n          const elapsed = performance.now() - t0;\n          stats.waitMs += elapsed;\n          lc.info?.(\n            `change stream reached ${msg.watermark} ` +\n              `after waiting ${Math.round(elapsed)} ms`,\n          );\n        }\n\n        if (\n          msg.tag === 'backfill' &&\n          msg.rowValues.length > 0 &&\n          msg.relation.rowKey.columns.length === 0\n        ) {\n          throw new MissingRowKeyError(state.request);\n        }\n\n        // Reserve the changeStreamer if not in a transaction.\n        if (backfillTx === null) {\n          const t0 = performance.now();\n          backfillTx = await beginTxFor(msg);\n          stats.reserveMs += performance.now() - t0;\n          if (backfillTx === null) {\n            lc.info?.(\n              `backfill stream canceled: ${state.stopped.signal.reason}`,\n              state.request,\n            );\n            this.#checkAndStartBackfill(); // start the next backfill if present\n            return; // this backfill is canceled\n          }\n        }\n\n        // `await` to allow the change streamer to exert back pressure\n        // on backfills.\n        const t0 = performance.now();\n        await changeStream.push(['data', msg]);\n        stats.pushMs += performance.now() - t0;\n        uncommittedBytes += byteSize;\n        stats.messages++;\n        stats.bytes += byteSize;\n\n        this.#maybeLogBackfillStats(lc, stats);\n      }\n\n      // Flush any final tx and release the stream.\n      if (backfillTx) {\n        commitTx();\n      }\n      lc.debug?.(`backfill stream exited`);\n    } finally {\n      stopProducer = true;\n      credits.enqueue(1);\n      // The producer can potentially be hung on upstream I/O.\n      // TODO: Thread AbortSignal logic into the backfillStreamer.\n      if (!this.#canceled) {\n        await producer;\n      }\n      this.#controller.signal.removeEventListener('abort', onCancel);\n      this.#logBackfillStats(lc, stats);\n    }\n  }\n\n  #maybeLogBackfillStats(lc: LogContext, stats: BackfillTimingStats) {\n    const now = performance.now();\n    if (now - stats.lastLogged >= BACKFILL_STATS_LOG_INTERVAL_MS) {\n      this.#logBackfillStats(lc, stats);\n      stats.lastLogged = now;\n    }\n  }\n\n  /**\n   * Logs where the backfill spent its time. `produceMs` (upstream read +\n   * parse/decode) accrues on the prefetch producer concurrently with the\n   * consumer's `pushMs` (downstream backpressure), so the two can overlap and\n   * their sum can exceed `elapsedMs`. `reserveMs` is time spent (re)acquiring\n   * the change-stream reservation, and `stallMs` is time the consumer waited on\n   * an empty prefetch queue (i.e. blocked on the upstream) — a large `stallMs`\n   * indicates an upstream-bound backfill. `waitMs` is time spent waiting\n   * (without the reservation) for the change stream to reach the backfill\n   * watermark, during which the upstream snapshot is held open.\n   */\n  #logBackfillStats(lc: LogContext, stats: BackfillTimingStats) {\n    const elapsed = performance.now() - stats.start;\n    lc.info?.(`backfill timing`, {\n      messages: stats.messages,\n      bytes: stats.bytes,\n      commits: stats.commits,\n      stalls: stats.stalls,\n      elapsedMs: Math.round(elapsed),\n      produceMs: Math.round(stats.produceMs),\n      pushMs: Math.round(stats.pushMs),\n      reserveMs: Math.round(stats.reserveMs),\n      stallMs: Math.round(stats.stallMs),\n      waitMs: Math.round(stats.waitMs),\n      throughputKBps:\n        elapsed > 0 ? Math.round(stats.bytes / 1024 / (elapsed / 1000)) : 0,\n    });\n  }\n\n  #backfillRunningFor(table: Identifier): RunningBackfillState | null {\n    const state = this.#runningBackfill;\n    return state?.request.table.schema === table.schema &&\n      state.request.table.name === table.name\n      ? state\n      : null;\n  }\n\n  /**\n   * Stops the running backfill for the specified `reason`. If `instance` is\n   * specified, the running backfill is stopped only if it is that instance.\n   * This allows the running backfill itself to clear backfill state without\n   * accidentally stopping a different (e.g. subsequent) backfill.\n   */\n  #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n    const backfill = this.#runningBackfill;\n    if (backfill && backfill === (instance ?? backfill)) {\n      backfill.stopped.abort(reason);\n      this.#runningBackfill = null;\n      reason && this.#lc.info?.(`canceling backfill:`, reason);\n    }\n  }\n\n  #setRequiredBackfill(source: string, req: BackfillRequest) {\n    const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n    this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n    this.#requiredBackfills.set(req.table, req);\n  }\n\n  #deleteRequiredBackfill(source: string, id: Identifier) {\n    const req = this.#requiredBackfills.get(id);\n    if (req) {\n      const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n      this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n      this.#requiredBackfills.delete(id);\n    }\n  }\n\n  /**\n   * Implements {@link Listener.onChange()}, invoked by the\n   * {@link ChangeStreamMultiplexer}.\n   */\n  onChange(message: ChangeStreamMessage): void {\n    if (message[0] === 'begin') {\n      this.#currentTxWatermark = message[2].commitWatermark;\n      return;\n    }\n    if (message[0] === 'commit') {\n      this.#currentTxWatermark = null;\n      this.#setLastStatusWatermark(message[2]);\n      // Every commit is a candidate for starting the next backfill\n      // (if one is not currently running).\n      this.#checkAndStartBackfill();\n      return;\n    }\n    if (message[0] === 'status') {\n      this.#setLastStatusWatermark(message[2]);\n      return;\n    }\n    if (message[0] !== 'data') {\n      return;\n    }\n    const change = message[1];\n    const {tag} = change;\n    switch (tag) {\n      case 'update-table-metadata': {\n        const {table, new: metadata} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest) {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, metadata},\n          });\n          if (this.#backfillRunningFor(table)) {\n            this.#stopRunningBackfill(`TableMetadata updated`);\n          }\n        }\n        break;\n      }\n      case 'create-table': {\n        const {\n          spec: {schema, name},\n          metadata = null,\n          backfill,\n        } = change;\n\n        if (backfill) {\n          this.#setRequiredBackfill(tag, {\n            table: {schema, name, metadata},\n            columns: Object.fromEntries(\n              Object.entries(backfill).map(([col, id]) => [col, {id}]),\n            ),\n          });\n        }\n        break;\n      }\n      case 'rename-table': {\n        const {old, new: newTable} = change;\n        const backfillRequest = this.#requiredBackfills.get(old);\n        if (backfillRequest) {\n          const {schema, name} = newTable;\n          this.#deleteRequiredBackfill(tag, old);\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            table: {...backfillRequest.table, schema, name},\n          });\n          if (this.#backfillRunningFor(old)) {\n            this.#stopRunningBackfill(`table renamed`);\n          }\n        }\n        break;\n      }\n      case 'drop-table': {\n        const {id} = change;\n        const backfillRequest = this.#requiredBackfills.get(id);\n        if (backfillRequest) {\n          this.#deleteRequiredBackfill(tag, id);\n          if (this.#backfillRunningFor(id)) {\n            this.#stopRunningBackfill(`table dropped`);\n          }\n        }\n        break;\n      }\n      case 'add-column': {\n        const {\n          table,\n          tableMetadata: metadata = null,\n          column,\n          backfill,\n        } = change;\n        if (backfill) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (!backfillRequest) {\n            this.#setRequiredBackfill(tag, {\n              table: {...table, metadata},\n              columns: {[column.name]: {id: backfill}},\n            });\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              table: {...backfillRequest.table, metadata},\n              columns: {\n                ...backfillRequest.columns,\n                [column.name]: {id: backfill},\n              },\n            });\n            // Note: The running backfill need not be canceled if a\n            //   new column is added. The new column will be backfilled\n            //   by its own stream after the current backfill completes.\n          }\n        }\n        break;\n      }\n      case 'update-column': {\n        const {\n          table,\n          old: {name: oldName},\n          new: {name: newName},\n        } = change;\n        if (oldName !== newName) {\n          const backfillRequest = this.#requiredBackfills.get(table);\n          if (backfillRequest && oldName in backfillRequest.columns) {\n            const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: {...otherCols, [newName]: colSpec},\n            });\n            const backfill = this.#backfillRunningFor(table);\n            if (backfill && oldName in backfill.request.columns) {\n              this.#stopRunningBackfill(`column renamed`);\n            }\n          }\n        }\n        break;\n      }\n      case 'drop-column': {\n        const {table, column} = change;\n        const backfillRequest = this.#requiredBackfills.get(table);\n        if (backfillRequest && column in backfillRequest.columns) {\n          const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n          if (Object.keys(remaining).length === 0) {\n            this.#deleteRequiredBackfill(tag, table);\n          } else {\n            this.#setRequiredBackfill(tag, {\n              ...backfillRequest,\n              columns: remaining,\n            });\n          }\n          const backfill = this.#backfillRunningFor(table);\n          if (backfill && column in backfill.request.columns) {\n            this.#stopRunningBackfill(`column dropped`);\n          }\n        }\n        break;\n      }\n      case 'update': {\n        const {relation, key, new: row} = change;\n        const backfill = this.#backfillRunningFor(relation);\n        const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n        if (backfill?.request.table.metadata && key !== null) {\n          // A corner case that backfill is unable to correctly handle is\n          // when a row's key changes; this is decomposed into a delete\n          // of the old key and a set of the new key in the replica change\n          // log, at which point the backfill algorithm assumes that the\n          // (old) row is deleted but does not know to backfill the new row.\n          // In these corner cases, the current backfill is canceled and\n          // retried if its version precedes this update.\n          for (const col of Object.keys(\n            backfill.request.table.metadata.rowKey,\n          )) {\n            if (key[col] !== row[col]) {\n              backfill.minWatermark = txWatermark;\n              this.#lc.info?.(\n                `key for row as changed (col: ${col}). ` +\n                  `backfill data must not predate ${backfill.minWatermark}`,\n              );\n              break;\n            }\n          }\n        }\n        break;\n      }\n      case 'backfill-completed': {\n        const {relation, columns} = change;\n        const backfillRequest = this.#requiredBackfills.get(relation);\n        assert(\n          backfillRequest,\n          () => `No BackfillRequest completed backfill ${stringify(change)}`,\n        );\n        const remaining = Object.entries(backfillRequest.columns).filter(\n          ([col]) =>\n            !(columns.includes(col) || relation.rowKey.columns.includes(col)),\n        );\n        if (remaining.length === 0) {\n          this.#deleteRequiredBackfill(tag, relation);\n        } else {\n          this.#setRequiredBackfill(tag, {\n            ...backfillRequest,\n            columns: Object.fromEntries(remaining),\n          });\n        }\n        // Technically the backfill is already stopping, but this method\n        // cleans up the state that tracks it.\n        this.#stopRunningBackfill();\n        break;\n      }\n    }\n  }\n\n  cancel(): void {\n    this.#controller.abort('canceled');\n    this.#stopRunningBackfill(`change stream canceled`);\n    clearTimeout(this.#backfillRetryTimer);\n    this.#backfillRetryTimer = undefined;\n\n    // Wake up a backfill that is waiting for the change stream to reach a\n    // watermark. The change stream never will, and the backfill must be\n    // allowed to unwind (and release its upstream resources) rather than\n    // remain suspended forever.\n    for (const {reached} of this.#awaitingStatusWatermarks.splice(0)) {\n      reached();\n    }\n  }\n}\n\nabstract class BackfillStreamError extends Error {\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(\n      `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n        `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n      {cause},\n    );\n  }\n}\n\n/**\n * Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n *\n * In terms of backfill, however, non-empty tables without a row key **cannot**\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n *\n * The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n */\nclass MissingRowKeyError extends BackfillStreamError {\n  readonly name = 'MissingRowKeyError';\n\n  constructor(bf: BackfillRequest, cause?: unknown) {\n    super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n  }\n}\n\n/**\n * Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n  readonly name = 'SchemaIncompatibilityError';\n\n  constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n    super(bf, msg, cause);\n  }\n}\n"],"mappings":";;;;;;;;;;;;AA4BA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;;;;;;AAsBA,IAAM,yBAAyB;;;;;;;;AAS/B,IAAM,4BAA4B;;;;;;;;;;;;;;AAelC,IAAM,0BAA0B;;AAGhC,IAAM,iCAAiC;;AAGvC,IAAM,OAAO,OAAO,eAAe;;AAEnC,IAAM,OAAO,OAAO,eAAe;AAmCnC,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC;;;;;;;CAQA,cAAuB,IAAI,gBAAgB;CAE3C,IAAI,YAAY;EACd,OAAO,KAAK,YAAY,OAAO;CACjC;CAEA,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf,uBAAuB,wBACvB;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAK,kBAAkB;EACvB,KAAK,oBAAoB;EACzB,KAAK,cAAc;EACnB,KAAK,gBAAgB;EACrB,KAAK,gBAAgB;EACrB,KAAK,gBAAgB;EACrB,KAAK,wBAAwB;CAC/B;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAK,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAK,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAK,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAK,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAK,wBAAwB,MAAM,WAAW;GACjD,KAAK,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAK,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAK,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAK,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAK,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAK,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAK,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAK,aACN,CAAC,KAAK,uBACN,CAAC,KAAK,oBACN,KAAK,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAK,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IACZ;IACA,cAAc;IACd,SAAS,IAAI,gBAAgB;GAC/B;GACA,MAAM,KAAK,KAAK,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAK,mBAAmB;GACxB,KAAU,aAAa,IAAI,KAAK,CAAC,CAC9B,WAAW;IACV,KAAK,qBAAqB,mBAAmB,KAAK;IAClD,KAAK,gBAAgB,KAAK;GAC5B,CAAC,CAAC,CAGD,OAAM,MAAK;IACV,KAAK,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAK,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,IAAI,KAAK,WAAW;GAClB,KAAK,IAAI,QAAQ,iDAAiD,CAAC;GACnE;EACF;EACA,KAAK,IAAI,OACP,uCAAuC,KAAK,cAAc,MAC1D,CACF;EACA,KAAK,sBAAsB,iBAAiB;GAC1C,KAAK,sBAAsB,KAAA;GAC3B,KAAK,uBAAuB;EAC9B,GAAG,KAAK,aAAa;EAErB,KAAK,gBAAgB,KAAK,IAAI,KAAK,gBAAgB,GAAG,KAAK,aAAa;CAC1E;CAEA,MAAM,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAK;EAI1B,IAAI,aAA4B;EAChC,IAAI,mBAAmB;EAEvB,MAAM,QAA6B;GACjC,OAAO,YAAY,IAAI;GACvB,YAAY,YAAY,IAAI;GAC5B,UAAU;GACV,OAAO;GACP,SAAS;GACT,QAAQ;GACR,WAAW;GACX,QAAQ;GACR,WAAW;GACX,SAAS;GACT,QAAQ;EACV;;;;EAKA,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,QAAQ,OAAO,WACpB,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,CAAC,MAAM,QAAQ,OAAO,SAAS;KACjC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAK,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,KAAK,IAAI,WAOX,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAK;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;IAC/B,MAAM;GACR;GACA,aAAa;GACb,mBAAmB;EACrB;EASA,MAAM,QAAQ,IAAI,MAAY;EAC9B,MAAM,UAAU,IAAI,MAAS;EAC7B,KAAK,IAAI,IAAI,GAAG,IAAI,2BAA2B,KAC7C,QAAQ,QAAQ,CAAC;EAEnB,IAAI,eAAe;EAInB,MAAM,iBACJ,MAAM,iBAAiB,IAAI,WAAW,mBAAmB,CAAC;EAC5D,IAAI,CAAC,KAAK,WACR,KAAK,YAAY,OAAO,iBAAiB,SAAS,QAAQ;EAG5D,MAAM,YAAY,YAAY;GAC5B,IAAI;IACF,IAAI,KAAK,YAAY,IAAI;IACzB,WAAW,MAAM,SAAS,KAAK,kBAAkB,MAAM,OAAO,GAAG;KAC/D,MAAM,aAAa,YAAY,IAAI,IAAI;KACvC,MAAM,QAAQ,QAAQ;KACtB,IAAI,gBAAgB,KAAK,WAAW;MAClC,MAAM,iBAAiB,IAAI,WAAW,mBAAmB,CAAC;MAC1D;KACF;KACA,MAAM,QAAQ,KAAK;KACnB,KAAK,YAAY,IAAI;IACvB;IACA,MAAM,QAAQ,IAAI;GACpB,SAAS,GAAG;IACV,MAAM,iBAAiB,CAAC;GAC1B;EACF,EAAA,CAAG;EAIH,SAAc,YAAY,CAAC,CAAC;EAS5B,MAAM,WAAW,YAAoD;GAKnE,IAAI,eAAe,QAAQ,MAAM,KAAK,IAAI,GAAG;IAC3C,MAAM,IAAI,MAAM,QAAQ;IAExB,OADU,aAAa,UAAU,MAAM,IAAI;GAE7C;GAEA,SAAS;IACP,IAAI,eAAe,QAAQ,aAAa,YAAY,IAAI,GACtD,SAAS;IAEX,MAAM,IACJ,eAAe,OACX,MAAM,QAAQ,MAAM,uBAAuB,IAC3C,MAAM,QAAQ;IACpB,MAAM,KAAK,YAAY,IAAI;IAC3B,MAAM,IAAI,aAAa,UAAU,MAAM,IAAI;IAC3C,IAAI,MAAM,MAAM;KACd,MAAM,WAAW,YAAY,IAAI,IAAI;KACrC;IACF;IACA,IAAI,aAAa,SAAS;KACxB,MAAM,WAAW,YAAY,IAAI,IAAI;KACrC,MAAM;IACR;IACA,OAAO;GACT;EACF;EAEA,IAAI;GACF,SAAS;IACP,MAAM,OAAO,MAAM,SAAS;IAC5B,IAAI,SAAS,MACX;IAIF,QAAQ,QAAQ,CAAC;IAEjB,IAAI,KAAK,WAAW;KAIlB,GAAG,OAAO,kDAAkD;KAC5D;IACF;IACA,MAAM,EAAC,SAAS,KAAK,aAAY;IAOjC,MAAM,sBAAsB,KAAK,qBAC/B,IACA,IAAI,SACN;IAKA,IACE,eACC,aAAa,YAAY,IAAI,KAC5B,uBACA,oBAAoB,KAAK,wBAE3B,SAAS;IAGX,IAAI,qBAAqB;KACvB,MAAM,KAAK,YAAY,IAAI;KAG3B,MAAM,eAAe,qBAAqB,MAAM,QAAQ,MAAM;KAC9D,MAAM,UAAU,YAAY,IAAI,IAAI;KACpC,MAAM,UAAU;KAChB,GAAG,OACD,yBAAyB,IAAI,UAAU,iBACpB,KAAK,MAAM,OAAO,EAAE,IACzC;IACF;IAEA,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;IAI5C,IAAI,eAAe,MAAM;KACvB,MAAM,KAAK,YAAY,IAAI;KAC3B,aAAa,MAAM,WAAW,GAAG;KACjC,MAAM,aAAa,YAAY,IAAI,IAAI;KACvC,IAAI,eAAe,MAAM;MACvB,GAAG,OACD,6BAA6B,MAAM,QAAQ,OAAO,UAClD,MAAM,OACR;MACA,KAAK,uBAAuB;MAC5B;KACF;IACF;IAIA,MAAM,KAAK,YAAY,IAAI;IAC3B,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;IACrC,MAAM,UAAU,YAAY,IAAI,IAAI;IACpC,oBAAoB;IACpB,MAAM;IACN,MAAM,SAAS;IAEf,KAAK,uBAAuB,IAAI,KAAK;GACvC;GAGA,IAAI,YACF,SAAS;GAEX,GAAG,QAAQ,wBAAwB;EACrC,UAAU;GACR,eAAe;GACf,QAAQ,QAAQ,CAAC;GAGjB,IAAI,CAAC,KAAK,WACR,MAAM;GAER,KAAK,YAAY,OAAO,oBAAoB,SAAS,QAAQ;GAC7D,KAAK,kBAAkB,IAAI,KAAK;EAClC;CACF;CAEA,uBAAuB,IAAgB,OAA4B;EACjE,MAAM,MAAM,YAAY,IAAI;EAC5B,IAAI,MAAM,MAAM,cAAc,gCAAgC;GAC5D,KAAK,kBAAkB,IAAI,KAAK;GAChC,MAAM,aAAa;EACrB;CACF;;;;;;;;;;;;CAaA,kBAAkB,IAAgB,OAA4B;EAC5D,MAAM,UAAU,YAAY,IAAI,IAAI,MAAM;EAC1C,GAAG,OAAO,mBAAmB;GAC3B,UAAU,MAAM;GAChB,OAAO,MAAM;GACb,SAAS,MAAM;GACf,QAAQ,MAAM;GACd,WAAW,KAAK,MAAM,OAAO;GAC7B,WAAW,KAAK,MAAM,MAAM,SAAS;GACrC,QAAQ,KAAK,MAAM,MAAM,MAAM;GAC/B,WAAW,KAAK,MAAM,MAAM,SAAS;GACrC,SAAS,KAAK,MAAM,MAAM,OAAO;GACjC,QAAQ,KAAK,MAAM,MAAM,MAAM;GAC/B,gBACE,UAAU,IAAI,KAAK,MAAM,MAAM,QAAQ,QAAQ,UAAU,IAAK,IAAI;EACtE,CAAC;CACH;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAK;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAK;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,QAAQ,MAAM,MAAM;GAC7B,KAAK,mBAAmB;GACxB,UAAU,KAAK,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAK,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAK,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAK,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAK,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAK,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAK,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAK,sBAAsB,QAAQ,EAAE,CAAC;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAK,sBAAsB;GAC3B,KAAK,wBAAwB,QAAQ,EAAE;GAGvC,KAAK,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAK,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAK,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAK,oBAAoB,KAAK,GAChC,KAAK,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAK,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS,OAAO,YACd,OAAO,QAAQ,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,KAAK,EAAC,GAAE,CAAC,CAAC,CACzD;IACF,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAK,wBAAwB,KAAK,GAAG;KACrC,KAAK,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAK,oBAAoB,GAAG,GAC9B,KAAK,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAK,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAK,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAK,oBAAoB,EAAE,GAC7B,KAAK,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAK,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,EAAC,IAAI,SAAQ,EAAC;KACzC,CAAC;UAED,KAAK,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO,EAAC,IAAI,SAAQ;MAC9B;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAK,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAK,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAK,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,CAAC,CAAC,WAAW,GACpC,KAAK,wBAAwB,KAAK,KAAK;UAEvC,KAAK,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAK,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAK,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAK,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAK,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ,MAQzC;UAAA,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAK,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,CAAC,CAAC,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAK,wBAAwB,KAAK,QAAQ;SAE1C,KAAK,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAK,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAK,YAAY,MAAM,UAAU;EACjC,KAAK,qBAAqB,wBAAwB;EAClD,aAAa,KAAK,mBAAmB;EACrC,KAAK,sBAAsB,KAAA;EAM3B,KAAK,MAAM,EAAC,aAAY,KAAK,0BAA0B,OAAO,CAAC,GAC7D,QAAQ;CAEZ;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,CAAC,CAAC,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js

@@ -1,7 +1,7 @@
 import { assert } from "../../../../../shared/src/asserts.js";
+import { resolver } from "../../../../../shared/src/resolver.js";
 import { AbortError } from "../../../../../shared/src/abort-error.js";
 import { Subscription } from "../../../types/subscription.js";
-import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts
 /**
 * Facilitates cooperative multiplexing of transactions from

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-stream-multiplexer.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Source} from '../../../types/streams.ts';\nimport {Subscription} from '../../../types/subscription.ts';\nimport type {\n  ChangeStreamMessage,\n  DownstreamStatusMessage,\n} from '../protocol/current.ts';\n\nexport type Cancelable = {\n  cancel(): void;\n};\n\nexport type Listener = {\n  onChange(change: ChangeStreamMessage): void;\n};\n\ntype Waiter = {\n  producer: string;\n  startTime: number;\n  grantReservation: (watermark: string) => void;\n  rejectReservation: (err: Error) => void;\n};\n\n/**\n * Facilitates cooperative multiplexing of transactions from\n * multiple producers into a single change stream.\n */\nexport class ChangeStreamMultiplexer {\n  readonly #lc: LogContext;\n  readonly #sub: Subscription<ChangeStreamMessage>;\n  readonly #producers: Cancelable[] = [];\n  readonly #listeners: Listener[] = [];\n\n  /**\n   * The `#lastWatermark` tracks the watermark of the last transaction\n   * committed to stream. It is set to null when a producer has reserved\n   * the stream, and set to the new watermark when the producer releases\n   * the reservation.\n   */\n  #lastWatermark: string | null;\n\n  /**\n   * Tracks the queue of producers waiting for a reservation.\n   */\n  readonly #waiters: Waiter[] = [];\n\n  #canceled = false;\n\n  constructor(lc: LogContext, lastWatermark: string) {\n    this.#lc = lc;\n    this.#sub = Subscription.create<ChangeStreamMessage>({\n      cleanup: () => this.#cancel(),\n    });\n    this.#lastWatermark = lastWatermark;\n  }\n\n  #cancel() {\n    this.#canceled = true;\n    this.#producers.forEach(p => p.cancel());\n\n    // A producer awaiting a reservation would otherwise remain suspended\n    // forever, since the reservation held when the stream was canceled is\n    // never released. Reject the pending reservations so that the producers\n    // unwind and release the resources (e.g. upstream connections and\n    // transactions) that they were holding for the stream.\n    for (const {producer, rejectReservation} of this.#waiters.splice(0)) {\n      this.#lc.info?.(\n        `rejecting reservation request from ${producer}: stream canceled`,\n      );\n      rejectReservation(new AbortError('change stream canceled'));\n    }\n  }\n\n  addProducers(...p: Cancelable[]): this {\n    this.#producers.push(...p);\n    return this;\n  }\n\n  addListeners(...l: Listener[]): this {\n    this.#listeners.push(...l);\n    return this;\n  }\n\n  /**\n   * Called by a producer to \"reserve\" the exclusive right to push data\n   * changes to the stream. If the stream is currently reserved,\n   * the resulting Promise must be awaited.\n   *\n   * The producer must then {@link release()} the reservation when it sees\n   * fit.\n   *\n   * @param producer The name of the producer, purely for debugging output\n   */\n  reserve(producer: string): string | Promise<string> {\n    if (this.#canceled) {\n      return Promise.reject(new AbortError('change stream canceled'));\n    }\n    if (this.#lastWatermark !== null) {\n      // If the stream is not reserved, reserve it and return the\n      // watermark.\n      const lastWatermark = this.#lastWatermark;\n      this.#lastWatermark = null;\n      return lastWatermark;\n    }\n\n    // Otherwise, wait for the current reservation to be released.\n    const startTime = performance.now();\n    const {\n      promise,\n      resolve: grantReservation,\n      reject: rejectReservation,\n    } = resolver<string>();\n    this.#waiters.push({\n      producer,\n      startTime,\n      grantReservation,\n      rejectReservation,\n    });\n\n    return promise;\n  }\n\n  /**\n   * If there are producers currently awaiting a reservation, returns\n   * the duration (in milliseconds) of the oldest reservation request.\n   * Returns a negative number if there are no waiters.\n   */\n  waiterDelay(): number {\n    if (this.#waiters.length === 0) {\n      return -1;\n    }\n    return performance.now() - this.#waiters[0].startTime;\n  }\n\n  /**\n   * Called by a producer to release its reservation after committing its\n   * last transaction.\n   */\n  release(newWatermark: string) {\n    const waiter = this.#waiters.shift();\n    if (!waiter) {\n      this.#lastWatermark = newWatermark;\n    } else {\n      const {producer, startTime, grantReservation} = waiter;\n      grantReservation(newWatermark);\n      const elapsed = performance.now() - startTime;\n      this.#lc.info?.(\n        `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`,\n      );\n    }\n  }\n\n  /**\n   * `pushStatus()` can be called without a reservation, as it\n   * does not constitute a data change and can appear anywhere\n   * in the stream.\n   */\n  pushStatus(message: DownstreamStatusMessage) {\n    this.#listeners.forEach(l => l.onChange(message));\n    this.#sub.push(message);\n  }\n\n  /**\n   * `push()` must only be called by a producer after it has\n   * {@link reserve}d the stream.\n   */\n  push(message: ChangeStreamMessage): Promise<unknown> {\n    assert(\n      this.#lastWatermark === null,\n      `push() called without reserve()-ing the stream`,\n    );\n    this.#listeners.forEach(l => l.onChange(message));\n    return this.#sub.push(message).result;\n  }\n\n  fail(err: Error) {\n    this.#sub.fail(err);\n  }\n\n  asSource(): Source<ChangeStreamMessage> {\n    return this.#sub;\n  }\n}\n"],"mappings":";;;;;;;;;AA8BA,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA,aAAoC,CAAC;CACrC,aAAkC,CAAC;;;;;;;CAQnC;;;;CAKA,WAA8B,CAAC;CAE/B,YAAY;CAEZ,YAAY,IAAgB,eAAuB;EACjD,KAAK,MAAM;EACX,KAAK,OAAO,aAAa,OAA4B,EACnD,eAAe,KAAK,QAAQ,EAC9B,CAAC;EACD,KAAK,iBAAiB;CACxB;CAEA,UAAU;EACR,KAAK,YAAY;EACjB,KAAK,WAAW,SAAQ,MAAK,EAAE,OAAO,CAAC;EAOvC,KAAK,MAAM,EAAC,UAAU,uBAAsB,KAAK,SAAS,OAAO,CAAC,GAAG;GACnE,KAAK,IAAI,OACP,sCAAsC,SAAS,kBACjD;GACA,kBAAkB,IAAI,WAAW,wBAAwB,CAAC;EAC5D;CACF;CAEA,aAAa,GAAG,GAAuB;EACrC,KAAK,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;CAEA,aAAa,GAAG,GAAqB;EACnC,KAAK,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;;;;;;;;;;;CAYA,QAAQ,UAA4C;EAClD,IAAI,KAAK,WACP,OAAO,QAAQ,OAAO,IAAI,WAAW,wBAAwB,CAAC;EAEhE,IAAI,KAAK,mBAAmB,MAAM;GAGhC,MAAM,gBAAgB,KAAK;GAC3B,KAAK,iBAAiB;GACtB,OAAO;EACT;EAGA,MAAM,YAAY,YAAY,IAAI;EAClC,MAAM,EACJ,SACA,SAAS,kBACT,QAAQ,sBACN,SAAiB;EACrB,KAAK,SAAS,KAAK;GACjB;GACA;GACA;GACA;EACF,CAAC;EAED,OAAO;CACT;;;;;;CAOA,cAAsB;EACpB,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO,YAAY,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;CAC9C;;;;;CAMA,QAAQ,cAAsB;EAC5B,MAAM,SAAS,KAAK,SAAS,MAAM;EACnC,IAAI,CAAC,QACH,KAAK,iBAAiB;OACjB;GACL,MAAM,EAAC,UAAU,WAAW,qBAAoB;GAChD,iBAAiB,YAAY;GAC7B,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,KAAK,IAAI,OACP,GAAG,SAAS,UAAU,QAAQ,QAAQ,CAAC,EAAE,2BAC3C;EACF;CACF;;;;;;CAOA,WAAW,SAAkC;EAC3C,KAAK,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAChD,KAAK,KAAK,KAAK,OAAO;CACxB;;;;;CAMA,KAAK,SAAgD;EACnD,OACE,KAAK,mBAAmB,MACxB,gDACF;EACA,KAAK,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAChD,OAAO,KAAK,KAAK,KAAK,OAAO,CAAC,CAAC;CACjC;CAEA,KAAK,KAAY;EACf,KAAK,KAAK,KAAK,GAAG;CACpB;CAEA,WAAwC;EACtC,OAAO,KAAK;CACd;AACF"}
+{"version":3,"file":"change-stream-multiplexer.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {resolver} from '../../../../../shared/src/resolver.ts';\nimport type {Source} from '../../../types/streams.ts';\nimport {Subscription} from '../../../types/subscription.ts';\nimport type {\n  ChangeStreamMessage,\n  DownstreamStatusMessage,\n} from '../protocol/current.ts';\n\nexport type Cancelable = {\n  cancel(): void;\n};\n\nexport type Listener = {\n  onChange(change: ChangeStreamMessage): void;\n};\n\ntype Waiter = {\n  producer: string;\n  startTime: number;\n  grantReservation: (watermark: string) => void;\n  rejectReservation: (err: Error) => void;\n};\n\n/**\n * Facilitates cooperative multiplexing of transactions from\n * multiple producers into a single change stream.\n */\nexport class ChangeStreamMultiplexer {\n  readonly #lc: LogContext;\n  readonly #sub: Subscription<ChangeStreamMessage>;\n  readonly #producers: Cancelable[] = [];\n  readonly #listeners: Listener[] = [];\n\n  /**\n   * The `#lastWatermark` tracks the watermark of the last transaction\n   * committed to stream. It is set to null when a producer has reserved\n   * the stream, and set to the new watermark when the producer releases\n   * the reservation.\n   */\n  #lastWatermark: string | null;\n\n  /**\n   * Tracks the queue of producers waiting for a reservation.\n   */\n  readonly #waiters: Waiter[] = [];\n\n  #canceled = false;\n\n  constructor(lc: LogContext, lastWatermark: string) {\n    this.#lc = lc;\n    this.#sub = Subscription.create<ChangeStreamMessage>({\n      cleanup: () => this.#cancel(),\n    });\n    this.#lastWatermark = lastWatermark;\n  }\n\n  #cancel() {\n    this.#canceled = true;\n    this.#producers.forEach(p => p.cancel());\n\n    // A producer awaiting a reservation would otherwise remain suspended\n    // forever, since the reservation held when the stream was canceled is\n    // never released. Reject the pending reservations so that the producers\n    // unwind and release the resources (e.g. upstream connections and\n    // transactions) that they were holding for the stream.\n    for (const {producer, rejectReservation} of this.#waiters.splice(0)) {\n      this.#lc.info?.(\n        `rejecting reservation request from ${producer}: stream canceled`,\n      );\n      rejectReservation(new AbortError('change stream canceled'));\n    }\n  }\n\n  addProducers(...p: Cancelable[]): this {\n    this.#producers.push(...p);\n    return this;\n  }\n\n  addListeners(...l: Listener[]): this {\n    this.#listeners.push(...l);\n    return this;\n  }\n\n  /**\n   * Called by a producer to \"reserve\" the exclusive right to push data\n   * changes to the stream. If the stream is currently reserved,\n   * the resulting Promise must be awaited.\n   *\n   * The producer must then {@link release()} the reservation when it sees\n   * fit.\n   *\n   * @param producer The name of the producer, purely for debugging output\n   */\n  reserve(producer: string): string | Promise<string> {\n    if (this.#canceled) {\n      return Promise.reject(new AbortError('change stream canceled'));\n    }\n    if (this.#lastWatermark !== null) {\n      // If the stream is not reserved, reserve it and return the\n      // watermark.\n      const lastWatermark = this.#lastWatermark;\n      this.#lastWatermark = null;\n      return lastWatermark;\n    }\n\n    // Otherwise, wait for the current reservation to be released.\n    const startTime = performance.now();\n    const {\n      promise,\n      resolve: grantReservation,\n      reject: rejectReservation,\n    } = resolver<string>();\n    this.#waiters.push({\n      producer,\n      startTime,\n      grantReservation,\n      rejectReservation,\n    });\n\n    return promise;\n  }\n\n  /**\n   * If there are producers currently awaiting a reservation, returns\n   * the duration (in milliseconds) of the oldest reservation request.\n   * Returns a negative number if there are no waiters.\n   */\n  waiterDelay(): number {\n    if (this.#waiters.length === 0) {\n      return -1;\n    }\n    return performance.now() - this.#waiters[0].startTime;\n  }\n\n  /**\n   * Called by a producer to release its reservation after committing its\n   * last transaction.\n   */\n  release(newWatermark: string) {\n    const waiter = this.#waiters.shift();\n    if (!waiter) {\n      this.#lastWatermark = newWatermark;\n    } else {\n      const {producer, startTime, grantReservation} = waiter;\n      grantReservation(newWatermark);\n      const elapsed = performance.now() - startTime;\n      this.#lc.info?.(\n        `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`,\n      );\n    }\n  }\n\n  /**\n   * `pushStatus()` can be called without a reservation, as it\n   * does not constitute a data change and can appear anywhere\n   * in the stream.\n   */\n  pushStatus(message: DownstreamStatusMessage) {\n    this.#listeners.forEach(l => l.onChange(message));\n    this.#sub.push(message);\n  }\n\n  /**\n   * `push()` must only be called by a producer after it has\n   * {@link reserve}d the stream.\n   */\n  push(message: ChangeStreamMessage): Promise<unknown> {\n    assert(\n      this.#lastWatermark === null,\n      `push() called without reserve()-ing the stream`,\n    );\n    this.#listeners.forEach(l => l.onChange(message));\n    return this.#sub.push(message).result;\n  }\n\n  fail(err: Error) {\n    this.#sub.fail(err);\n  }\n\n  asSource(): Source<ChangeStreamMessage> {\n    return this.#sub;\n  }\n}\n"],"mappings":";;;;;;;;;AA8BA,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA,aAAoC,CAAC;CACrC,aAAkC,CAAC;;;;;;;CAQnC;;;;CAKA,WAA8B,CAAC;CAE/B,YAAY;CAEZ,YAAY,IAAgB,eAAuB;EACjD,KAAK,MAAM;EACX,KAAK,OAAO,aAAa,OAA4B,EACnD,eAAe,KAAK,QAAQ,EAC9B,CAAC;EACD,KAAK,iBAAiB;CACxB;CAEA,UAAU;EACR,KAAK,YAAY;EACjB,KAAK,WAAW,SAAQ,MAAK,EAAE,OAAO,CAAC;EAOvC,KAAK,MAAM,EAAC,UAAU,uBAAsB,KAAK,SAAS,OAAO,CAAC,GAAG;GACnE,KAAK,IAAI,OACP,sCAAsC,SAAS,kBACjD;GACA,kBAAkB,IAAI,WAAW,wBAAwB,CAAC;EAC5D;CACF;CAEA,aAAa,GAAG,GAAuB;EACrC,KAAK,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;CAEA,aAAa,GAAG,GAAqB;EACnC,KAAK,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;;;;;;;;;;;CAYA,QAAQ,UAA4C;EAClD,IAAI,KAAK,WACP,OAAO,QAAQ,OAAO,IAAI,WAAW,wBAAwB,CAAC;EAEhE,IAAI,KAAK,mBAAmB,MAAM;GAGhC,MAAM,gBAAgB,KAAK;GAC3B,KAAK,iBAAiB;GACtB,OAAO;EACT;EAGA,MAAM,YAAY,YAAY,IAAI;EAClC,MAAM,EACJ,SACA,SAAS,kBACT,QAAQ,sBACN,SAAiB;EACrB,KAAK,SAAS,KAAK;GACjB;GACA;GACA;GACA;EACF,CAAC;EAED,OAAO;CACT;;;;;;CAOA,cAAsB;EACpB,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO,YAAY,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;CAC9C;;;;;CAMA,QAAQ,cAAsB;EAC5B,MAAM,SAAS,KAAK,SAAS,MAAM;EACnC,IAAI,CAAC,QACH,KAAK,iBAAiB;OACjB;GACL,MAAM,EAAC,UAAU,WAAW,qBAAoB;GAChD,iBAAiB,YAAY;GAC7B,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,KAAK,IAAI,OACP,GAAG,SAAS,UAAU,QAAQ,QAAQ,CAAC,EAAE,2BAC3C;EACF;CACF;;;;;;CAOA,WAAW,SAAkC;EAC3C,KAAK,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAChD,KAAK,KAAK,KAAK,OAAO;CACxB;;;;;CAMA,KAAK,SAAgD;EACnD,OACE,KAAK,mBAAmB,MACxB,gDACF;EACA,KAAK,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAChD,OAAO,KAAK,KAAK,KAAK,OAAO,CAAC,CAAC;CACjC;CAEA,KAAK,KAAY;EACf,KAAK,KAAK,KAAK,GAAG;CACpB;CAEA,WAAwC;EACtC,OAAO,KAAK;CACd;AACF"}