@rocicorp/zero 1.11.0-canary.25 → 1.11.0-canary.26

package/out/zero-cache/src/server/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {InitCleanup} from '../services/change-source/common/init-cleanup.ts';\nimport type {PgChangeLogPurgeLocker} from '../services/change-source/common/replica-restore.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {\n  PurgeLocker,\n  type PurgeLock,\n} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {\n  deleteLitestreamMetaDir,\n  startReplicaBackupProcess,\n} from '../services/litestream/commands.ts';\nimport type {WalKeeper} from '../services/litestream/wal-keeper.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  // Startup-time client used for for change-streamer initialization\n  // and handoff / takeover. Steady-state clients are managed by the\n  // change-streamer (Storer) implementation.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n\n  let purgeLock = null as PurgeLock | null;\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  let waitForFirstBackupBeforeServing = false;\n  let newBackupLineage = false;\n  let walKeeper: WalKeeper | undefined;\n  for (const first of [true, false]) {\n    // Resources acquired by an initialization attempt (e.g. a claimed\n    // replication slot, the purge lock) to be released if the attempt fails.\n    const cleanup = new InitCleanup(lc);\n    // When restoring from litestream, a lock is acquired to prevent change-log\n    // purges. This ensures that (this) change-streamer will be able to resume\n    // from the backup. The lock is acquired by the change source initialization\n    // just before restoring (i.e. after a replication slot is created, if\n    // applicable), and released once this change-streamer takes over the\n    // change-log.\n    const acquirePurgeLock: PgChangeLogPurgeLocker | undefined =\n      pgChangeLogEnabled && litestream.backupURL && litestream.executable\n        ? async (slotWatermark?: string) => {\n            const lock = await new PurgeLocker(lc, shard, changeDB).acquire(\n              slotWatermark,\n            );\n            purgeLock = lock === 'behind-slot' ? null : lock;\n            if (purgeLock) {\n              const acquired = purgeLock;\n              cleanup.onFailure('purge lock', () => {\n                purgeLock = null;\n                return acquired.release();\n              });\n            }\n            return lock;\n          }\n        : undefined;\n\n    const restoreOptions = {litestream, acquirePurgeLock, cleanup};\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        pgReplicationEpoch: epoch,\n        pgReplicationSlotPerReplica: slotPerReplica,\n        pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n      } = upstream;\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n        newBackupLineage: initNewBackupLineage,\n        walKeeper: initWalKeeper,\n      } = upstream.type === 'pg'\n        ? await initializePostgresChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            {\n              ...initialSync,\n              replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n            },\n            context,\n            replicationLag.reportIntervalMs,\n            restoreOptions,\n            {\n              epoch,\n              slotPerReplica,\n              inactiveReplicaGracePeriodMs,\n              backupV5: litestream.backupUsingV5,\n            },\n            upstream.pgStreamInboundTimeoutMs,\n          )\n        : await initializeCustomChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            context,\n            restoreOptions,\n          );\n      walKeeper = initWalKeeper;\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n              replicaFile: replica.file,\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogEnabled,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      newBackupLineage = initNewBackupLineage;\n      break;\n    } catch (e) {\n      // Release the resources acquired by the failed attempt, e.g.:\n      // * The claimed replication slot, which would otherwise remain active\n      //   indefinitely, preventing a retry from resuming (or forking) it.\n      // * The purge lock. This is safe because the purge lock exists to\n      //   preserve change-log entries so the new change-streamer can resume\n      //   from the backup replica's watermark. An AutoResetSignal means we\n      //   can't resume from the backup replica (e.g. its replication slot is\n      //   gone), so the change-log entries the lock was protecting are no\n      //   longer needed. The retry performs a fresh initial sync with a new\n      //   replication slot, independent of the old change-log. Releasing is\n      //   also necessary to avoid a self-deadlock when CHANGE_DB ==\n      //   UPSTREAM_DB: CREATE_REPLICATION_SLOT waits for all older\n      //   transactions to finish, including this lock's open transaction.\n      // * The WAL keeper of a restore that prepared a backup lineage to\n      //   continue, which would otherwise hold the (deleted) replica open.\n      await cleanup.release();\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  const processes = new ProcessManager(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  if (backupURL) {\n    lc.info?.('setting up backup to', backupURL);\n    litestream.backupURL = backupURL;\n    if (newBackupLineage) {\n      deleteLitestreamMetaDir(replica.file);\n    }\n    const {promise: backupStarted, resolve} = resolver();\n\n    // Start a backup replicator and corresponding litestream backup process.\n    const backupReplicator = processes\n      .addWorker(\n        childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n        'supporting',\n        'backup-replicator',\n      )\n      // Wait for the replicator's first message (i.e. \"ready\") before starting\n      // litestream backup in order to avoid contending on the sqlite lock\n      // when the replicator first prepares the db file.\n      .once('message', () => {\n        processes.addSubprocess(\n          startReplicaBackupProcess(lc, litestream, replica.file),\n          'supporting',\n          'litestream',\n        );\n        resolve();\n      });\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n    await backupStarted;\n  }\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n  });\n\n  let readinessGate = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    readinessGate = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n    });\n  }\n  // By the initial backup, litestream holds the restored WAL itself.\n  void readinessGate.then(\n    () => walKeeper?.release('initial backup confirmed'),\n    () => walKeeper?.release('initial backup failed'),\n  );\n\n  // Create the broadcast facade once: each broadcastWorker() adds permanent\n  // 'message' forwarders to every sub worker, so creating one per /profz\n  // request would leak a forwarder per request.\n  const profileWorker =\n    profileSubWorkers.length > 0\n      ? broadcastWorker(profileSubWorkers)\n      : undefined;\n  const getProfileWorker = profileWorker\n    ? () => Promise.resolve(profileWorker)\n    : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      // The startup delay is only relevant when taking over the PG change-log\n      // (RMv1 and RMv1.5), and is disabled for RMv2.\n      startupDelayMs: pgChangeLogEnabled ? startupDelayMs : 0,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  try {\n    await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'change-streamer');\n  } finally {\n    walKeeper?.release('shutting down');\n    await processes.shutdown();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        await publishCriticalEvent(\n          lc,\n          replicationStatusError(lc, 'Initializing', e),\n        );\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAElD,IAAI,YAAY;CAChB,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,IAAI,mBAAmB;CACvB,IAAI;CACJ,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAAG;EAGjC,MAAM,UAAU,IAAI,YAAY,EAAE;EAyBlC,MAAM,iBAAiB;GAAC;GAAY,kBAjBlC,sBAAsB,WAAW,aAAa,WAAW,aACrD,OAAO,kBAA2B;IAChC,MAAM,OAAO,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QACtD,aACF;IACA,YAAY,SAAS,gBAAgB,OAAO;IAC5C,IAAI,WAAW;KACb,MAAM,WAAW;KACjB,QAAQ,UAAU,oBAAoB;MACpC,YAAY;MACZ,OAAO,SAAS,QAAQ;KAC1B,CAAC;IACH;IACA,OAAO;GACT,IACA,KAAA;GAEgD;EAAO;EAC7D,IAAI;GAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;GACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,4BACA,kBAAkB,sBAClB,WAAW,kBACT,SAAS,SAAS,OAClB,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;IACE,GAAG;IACH,yBAAyB,SAAS;IAClC,6BAA6B,SAAS;GACxC,GACA,SACA,eAAe,kBACf,gBACA;IACE;IACA;IACA;IACA,UAAU,WAAW;GACvB,GACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;GACJ,YAAY;GAEZ,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;GAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;IACE,WAAW;IACX,mBAAmB,WAAW,gBAAgB,OAAO;IACrD,aAAa,QAAQ;GACvB,IACA,MACJ,WACA,aAAa,OACb;IACE;IACA;IACA;IACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;IACN,oBAAoB,OAAO;IAC3B,oBAAoB,OAAO;IAC3B,eAAe;KACb,eAAe,kBAAkB,QAAQ,IAAI;KAC7C,eAAe;KACf,kBAAkB;IACpB;IAIA,uBAAuB,yBACnB;KACE,aAAa,QAAQ;KACrB,UAAU;MAGR,OAAO;MACP,YAAY,kBAAkB;MAC9B;KACF;IACF,IACA,KAAA;IAIJ,sBAAsB,yBAClB;KACE,aAAa;KACb,WAAW;IACb,IACA,KAAA;IAEJ,wBACE,sBAAsB,2BAClB;KACE,aAAa,QAAQ;KACrB,gBAAgB;KAChB,aAAa;KACb,eAAe;IACjB,IACA,KAAA;IAIN,sBACE,wBAAwB,UACpB;KACE,aAAa;KACb,iBAAiB;KACjB,aAAa;IACf,IACA,KAAA;GACR,GACA,UACF;GACA,YAAY;GACZ,kCAAkC;GAClC,mBAAmB;GACnB;EACF,SAAS,GAAG;GAgBV,MAAM,QAAQ,QAAQ;GACtB,IAAI,SAAS,aAAa,iBAAiB;IACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;IAGhD,aAAa,QAAQ,IAAI;IAMzB,kBAAkB,QAAQ,IAAI;IAC9B;GACF;GACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;GAEF,MAAM;EACR;CACF;CAEA,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,MAAM,oBAA8B,CAAC;CACrC,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,IAAI,kBACF,wBAAwB,QAAQ,IAAI;EAEtC,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,MAAM,mBAAmB,UACtB,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;EACA,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,cAAmB,WACX,WAAW,QAAQ,0BAA0B,SAC7C,WAAW,QAAQ,uBAAuB,CAClD;CAKA,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAGA,gBAAgB,qBAAqB,iBAAiB;EACtD;EACA;EACA,kBAdqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAaF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,WAAW,QAAQ,eAAe;EAClC,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {InitCleanup} from '../services/change-source/common/init-cleanup.ts';\nimport type {PgChangeLogPurgeLocker} from '../services/change-source/common/replica-restore.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {\n  PurgeLocker,\n  type PurgeLock,\n} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {deleteLitestreamMetaDir} from '../services/litestream/commands.ts';\nimport type {WalKeeper} from '../services/litestream/wal-keeper.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  // Startup-time client used for for change-streamer initialization\n  // and handoff / takeover. Steady-state clients are managed by the\n  // change-streamer (Storer) implementation.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n\n  let purgeLock = null as PurgeLock | null;\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  let waitForFirstBackupBeforeServing = false;\n  let newBackupLineage = false;\n  let walKeeper: WalKeeper | undefined;\n  for (const first of [true, false]) {\n    // Resources acquired by an initialization attempt (e.g. a claimed\n    // replication slot, the purge lock) to be released if the attempt fails.\n    const cleanup = new InitCleanup(lc);\n    // When restoring from litestream, a lock is acquired to prevent change-log\n    // purges. This ensures that (this) change-streamer will be able to resume\n    // from the backup. The lock is acquired by the change source initialization\n    // just before restoring (i.e. after a replication slot is created, if\n    // applicable), and released once this change-streamer takes over the\n    // change-log.\n    const acquirePurgeLock: PgChangeLogPurgeLocker | undefined =\n      pgChangeLogEnabled && litestream.backupURL && litestream.executable\n        ? async (slotWatermark?: string) => {\n            const lock = await new PurgeLocker(lc, shard, changeDB).acquire(\n              slotWatermark,\n            );\n            purgeLock = lock === 'behind-slot' ? null : lock;\n            if (purgeLock) {\n              const acquired = purgeLock;\n              cleanup.onFailure('purge lock', () => {\n                purgeLock = null;\n                return acquired.release();\n              });\n            }\n            return lock;\n          }\n        : undefined;\n\n    const restoreOptions = {litestream, acquirePurgeLock, cleanup};\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        pgReplicationEpoch: epoch,\n        pgReplicationSlotPerReplica: slotPerReplica,\n        pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n      } = upstream;\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n        newBackupLineage: initNewBackupLineage,\n        walKeeper: initWalKeeper,\n      } = upstream.type === 'pg'\n        ? await initializePostgresChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            {\n              ...initialSync,\n              replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n            },\n            context,\n            replicationLag.reportIntervalMs,\n            restoreOptions,\n            {\n              epoch,\n              slotPerReplica,\n              inactiveReplicaGracePeriodMs,\n              backupV5: litestream.backupUsingV5,\n            },\n            upstream.pgStreamInboundTimeoutMs,\n          )\n        : await initializeCustomChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            context,\n            restoreOptions,\n          );\n      walKeeper = initWalKeeper;\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n              replicaFile: replica.file,\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogEnabled,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      newBackupLineage = initNewBackupLineage;\n      break;\n    } catch (e) {\n      // Release the resources acquired by the failed attempt, e.g.:\n      // * The claimed replication slot, which would otherwise remain active\n      //   indefinitely, preventing a retry from resuming (or forking) it.\n      // * The purge lock. This is safe because the purge lock exists to\n      //   preserve change-log entries so the new change-streamer can resume\n      //   from the backup replica's watermark. An AutoResetSignal means we\n      //   can't resume from the backup replica (e.g. its replication slot is\n      //   gone), so the change-log entries the lock was protecting are no\n      //   longer needed. The retry performs a fresh initial sync with a new\n      //   replication slot, independent of the old change-log. Releasing is\n      //   also necessary to avoid a self-deadlock when CHANGE_DB ==\n      //   UPSTREAM_DB: CREATE_REPLICATION_SLOT waits for all older\n      //   transactions to finish, including this lock's open transaction.\n      // * The WAL keeper of a restore that prepared a backup lineage to\n      //   continue, which would otherwise hold the (deleted) replica open.\n      await cleanup.release();\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  const processes = new ProcessManager(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  const {promise: replicatorReady, resolve} = resolver();\n  if (!backupURL) {\n    resolve(); // No backup replicator to wait for, so resolve immediately.\n  } else {\n    lc.info?.('setting up backup to', backupURL);\n    config.litestream.backupURLOverride = backupURL;\n    if (newBackupLineage) {\n      deleteLitestreamMetaDir(replica.file);\n    }\n    // Start a backup replicator, which starts up the corresponding\n    // litestream backup process.\n    const backupReplicator = processes\n      .addWorker(\n        childWorker(\n          REPLICATOR_URL,\n          {...env, ZERO_LITESTREAM_BACKUP_URL_OVERRIDE: backupURL},\n          'backup' satisfies ReplicaFileMode,\n        ),\n        'supporting',\n        'backup-replicator',\n      )\n      .onceMessageType('ready', () => resolve());\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n  }\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n  });\n\n  let backupReady = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    backupReady = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n    });\n  }\n  // By the initial backup, litestream holds the restored WAL itself.\n  void backupReady.then(\n    () => walKeeper?.release('initial backup confirmed'),\n    () => walKeeper?.release('initial backup failed'),\n  );\n\n  // In RMv2, readiness additionally waits for the backup-replicator to catch\n  // up to the replication stream. This is not done in RMv1 (i.e. when the\n  // PG change-log is enabled), in which the change-streamer itself is only\n  // started after the readinessGate (plus a startup delay), and thus the\n  // backup-replicator cannot catch up before then.\n  const readinessGate = pgChangeLogEnabled\n    ? backupReady\n    : Promise.all([backupReady, replicatorReady]);\n\n  // Create the broadcast facade once: each broadcastWorker() adds permanent\n  // 'message' forwarders to every sub worker, so creating one per /profz\n  // request would leak a forwarder per request.\n  const profileWorker =\n    profileSubWorkers.length > 0\n      ? broadcastWorker(profileSubWorkers)\n      : undefined;\n  const getProfileWorker = profileWorker\n    ? () => Promise.resolve(profileWorker)\n    : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      // The startup delay is only relevant when taking over the PG change-log\n      // (RMv1 and RMv1.5), and is disabled for RMv2.\n      startupDelayMs: pgChangeLogEnabled ? startupDelayMs : 0,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  try {\n    await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'change-streamer');\n  } finally {\n    walKeeper?.release('shutting down');\n    await processes.shutdown();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        await publishCriticalEvent(\n          lc,\n          replicationStatusError(lc, 'Initializing', e),\n        );\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAElD,IAAI,YAAY;CAChB,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,IAAI,mBAAmB;CACvB,IAAI;CACJ,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAAG;EAGjC,MAAM,UAAU,IAAI,YAAY,EAAE;EAyBlC,MAAM,iBAAiB;GAAC;GAAY,kBAjBlC,sBAAsB,WAAW,aAAa,WAAW,aACrD,OAAO,kBAA2B;IAChC,MAAM,OAAO,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QACtD,aACF;IACA,YAAY,SAAS,gBAAgB,OAAO;IAC5C,IAAI,WAAW;KACb,MAAM,WAAW;KACjB,QAAQ,UAAU,oBAAoB;MACpC,YAAY;MACZ,OAAO,SAAS,QAAQ;KAC1B,CAAC;IACH;IACA,OAAO;GACT,IACA,KAAA;GAEgD;EAAO;EAC7D,IAAI;GAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;GACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,4BACA,kBAAkB,sBAClB,WAAW,kBACT,SAAS,SAAS,OAClB,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;IACE,GAAG;IACH,yBAAyB,SAAS;IAClC,6BAA6B,SAAS;GACxC,GACA,SACA,eAAe,kBACf,gBACA;IACE;IACA;IACA;IACA,UAAU,WAAW;GACvB,GACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;GACJ,YAAY;GAEZ,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;GAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;IACE,WAAW;IACX,mBAAmB,WAAW,gBAAgB,OAAO;IACrD,aAAa,QAAQ;GACvB,IACA,MACJ,WACA,aAAa,OACb;IACE;IACA;IACA;IACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;IACN,oBAAoB,OAAO;IAC3B,oBAAoB,OAAO;IAC3B,eAAe;KACb,eAAe,kBAAkB,QAAQ,IAAI;KAC7C,eAAe;KACf,kBAAkB;IACpB;IAIA,uBAAuB,yBACnB;KACE,aAAa,QAAQ;KACrB,UAAU;MAGR,OAAO;MACP,YAAY,kBAAkB;MAC9B;KACF;IACF,IACA,KAAA;IAIJ,sBAAsB,yBAClB;KACE,aAAa;KACb,WAAW;IACb,IACA,KAAA;IAEJ,wBACE,sBAAsB,2BAClB;KACE,aAAa,QAAQ;KACrB,gBAAgB;KAChB,aAAa;KACb,eAAe;IACjB,IACA,KAAA;IAIN,sBACE,wBAAwB,UACpB;KACE,aAAa;KACb,iBAAiB;KACjB,aAAa;IACf,IACA,KAAA;GACR,GACA,UACF;GACA,YAAY;GACZ,kCAAkC;GAClC,mBAAmB;GACnB;EACF,SAAS,GAAG;GAgBV,MAAM,QAAQ,QAAQ;GACtB,IAAI,SAAS,aAAa,iBAAiB;IACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;IAGhD,aAAa,QAAQ,IAAI;IAMzB,kBAAkB,QAAQ,IAAI;IAC9B;GACF;GACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;GAEF,MAAM;EACR;CACF;CAEA,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,MAAM,oBAA8B,CAAC;CACrC,MAAM,EAAC,SAAS,iBAAiB,YAAW,SAAS;CACrD,IAAI,CAAC,WACH,QAAQ;MACH;EACL,GAAG,OAAO,wBAAwB,SAAS;EAC3C,OAAO,WAAW,oBAAoB;EACtC,IAAI,kBACF,wBAAwB,QAAQ,IAAI;EAItC,MAAM,mBAAmB,UACtB,UACC,YACE,gBACA;GAAC,GAAG;GAAK,qCAAqC;EAAS,GACvD,QACF,GACA,cACA,mBACF,EACC,gBAAgB,eAAe,QAAQ,CAAC;EAC3C,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;CACF;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,cAAc;CAClB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAc,cAAc,oBAAoB,EAAE,WAAW;GAC3D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,YAAiB,WACT,WAAW,QAAQ,0BAA0B,SAC7C,WAAW,QAAQ,uBAAuB,CAClD;CAOA,MAAM,gBAAgB,qBAClB,cACA,QAAQ,IAAI,CAAC,aAAa,eAAe,CAAC;CAK9C,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAGA,gBAAgB,qBAAqB,iBAAiB;EACtD;EACA;EACA,kBAdqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAaF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,WAAW,QAAQ,eAAe;EAClC,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}

package/out/zero-cache/src/server/replicator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAGA,OAAO,EAAiB,UAAU,EAAC,MAAM,kBAAkB,CAAC;AA0C5D,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAML,KAAK,OAAO,EACb,MAAM,0BAA0B,CAAC;AAOlC,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA4Hf;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,QAwDjB"}
+{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAGA,OAAO,EAAiB,UAAU,EAAC,MAAM,kBAAkB,CAAC;AA+C5D,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAML,KAAK,OAAO,EACb,MAAM,0BAA0B,CAAC;AAOlC,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAmJf;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,QAwDjB"}

package/out/zero-cache/src/server/replicator.js

@@ -8,13 +8,13 @@ import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { Database } from "../../../zqlite/src/db.js";
 import { getOrCreateGauge } from "../observability/metrics.js";
 import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruption.js";
-import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
+import { ProcessManager, exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { installProfileHandler } from "../types/profiler.js";
 import { initEventSink } from "../observability/events.js";
 import { ReplicationStatusPublisher } from "../services/replicator/replication-status.js";
 import { litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreRuns } from "../services/litestream/metrics.js";
-import { tryRestore } from "../services/litestream/commands.js";
+import { startReplicaBackupProcess, tryRestore } from "../services/litestream/commands.js";
 import { ChangeStreamerHttpClient } from "../services/change-streamer/change-streamer-http.js";
 import { sqliteFileBytes } from "../services/replicator/sqlite-change-log-observability.js";
 import { startOtelAuto } from "./otel-start.js";
@@ -47,7 +47,7 @@ async function runWorker(parent, env, ...args) {
 	}, config.sqliteCorruptionChecks);
 	initEventSink(lc, config);
 	const shard = getShardConfig(config);
-	const { taskID, change, changeStreamer: { port, uri: changeStreamerURI = runningLocalChangeStreamer ? `http://localhost:${port}/` : void 0 } } = config;
+	const { taskID, change, changeStreamer: { port, uri: changeStreamerURI = runningLocalChangeStreamer ? `http://localhost:${port}/` : void 0 }, replicationLag: { initialReadinessThresholdMs, maxReadinessDelayMs } } = config;
 	const changeStreamer = new ChangeStreamerHttpClient(lc, shard, change.db, changeStreamerURI);
 	let initialConnection;
 	if (fileMode === "serving" && !runningLocalChangeStreamer && (config.litestream.executable || config.litestream.executableV5)) initialConnection = await restoreReplica(lc, config, changeStreamer);
@@ -66,14 +66,24 @@ async function runWorker(parent, env, ...args) {
 	} : null;
 	const workerClient = new ThreadWriteWorkerClient();
 	await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);
-	const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null, initialConnection);
+	const processes = mode === "backup" ? new ProcessManager(lc, parent) : void 0;
+	processes?.addSubprocess(startReplicaBackupProcess(lc, config.litestream, dbPath), "supporting", "litestream");
+	const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null, {
+		readinessThresholdMs: initialReadinessThresholdMs,
+		maxReadinessDelayMs
+	}, initialConnection);
 	setUpMessageHandlers(lc, replicator, parent);
-	const running = runUntilKilled(lc, parent, replicator);
-	for await (const _ of replicator.subscribe()) {
+	replicator.ready().then(() => {
 		parent.send(["ready", { ready: true }]);
-		break;
+	});
+	if (!processes) return runUntilKilled(lc, parent, replicator);
+	try {
+		await runUntilKilled(lc, parent, replicator);
+	} catch (err) {
+		processes.logErrorAndExit(err, workerName);
+	} finally {
+		await processes.shutdown();
 	}
-	return running;
 }
 function setupMetrics(lc, file, walMode, pageSize) {
 	let db;

package/out/zero-cache/src/server/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n  ChangeStreamerHttpClient,\n  type ReservationFollowup,\n  type SnapshotReserver,\n} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  tryRestore,\n  type RestoreAttempt,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {RestoreProgressReporter} from '../services/litestream/restore-progress.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n  ThreadWriteWorkerClient,\n  type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  installProfileHandler(parent, workerName);\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  //\n  // The reservation's connection (`initialConnection`) is threaded through to\n  // IncrementalSyncer's first subscribe(), so the replication-manager that\n  // reserved the change log for this restore is the one that serves the\n  // subscription (single-connection handoff).\n  let initialConnection: ReservationFollowup | undefined;\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    initialConnection = await restoreReplica(lc, config, changeStreamer);\n  }\n\n  const {\n    file: dbPath,\n    walMode,\n    pageSize,\n  } = await setupReplica(lc, fileMode, config.replica);\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode, pageSize);\n\n  // Create the write worker for async SQLite writes. Enable write-path\n  // checkpoint backpressure only for the backup replicator on litestream v5,\n  // which owns the replica litestream backs up.\n  const pragmas = getPragmaConfig(fileMode);\n  const checkpoint: ForceCheckpointConfig | null =\n    mode === 'backup' &&\n    config.litestream.backupURL &&\n    config.litestream.backupUsingV5 &&\n    config.litestream.forceCheckpointThresholdMB\n      ? {\n          checkpointThresholdPages:\n            (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n            pageSize,\n          maxWalPages: !config.litestream.maxWalSizeMB\n            ? undefined\n            : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n        }\n      : null;\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n    initialConnection,\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  const running = runUntilKilled(lc, parent, replicator);\n\n  // Signal readiness once the first ReplicaVersionReady notification is received.\n  for await (const _ of replicator.subscribe()) {\n    parent.send(['ready', {ready: true}]);\n    break;\n  }\n\n  return running;\n}\n\nexport function setupMetrics(\n  lc: LogContext,\n  file: string,\n  walMode: WalMode,\n  pageSize: number,\n) {\n  let db: Database | undefined;\n  const getDb = () => {\n    if (!db) {\n      try {\n        db = new Database(lc, file, {readonly: true});\n      } catch (e) {\n        lc.warn?.(`unable to open ${file} for wal metrics`, e);\n      }\n    }\n    return db;\n  };\n\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  getOrCreateGauge('replica', 'wal_active_bytes', {\n    description:\n      `The actual size of active WAL frames currently in use ` +\n      `(excluding pre-allocated or reclaimed space).`,\n    unit: 'bytes',\n  }).addCallback(observeWalActiveBytes(lc, getDb, pageSize));\n\n  getOrCreateGauge('replica', 'wal_uncheckpointed_bytes', {\n    description:\n      `The size of un-checkpointed WAL frames that have not yet ` +\n      `been backfilled into the main database.`,\n    unit: 'bytes',\n  }).addCallback(observeWalUncheckpointedBytes(lc, getDb, pageSize));\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeWalActiveBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log}] = db.pragma<{log: number}>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number') {\n        o.observe(Math.max(0, log) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.('unable to inspect wal_checkpoint for active bytes metric', e);\n    }\n  };\n}\n\nfunction observeWalUncheckpointedBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log, checkpointed}] = db.pragma<{\n        log: number;\n        checkpointed: number;\n      }>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number' && typeof checkpointed === 'number') {\n        o.observe(Math.max(0, log - checkpointed) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.(\n        'unable to inspect wal_checkpoint for uncheckpointed bytes metric',\n        e,\n      );\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// View-syncers wait indefinitely for the replication-manager to publish a\n// restorable backup. On a fresh stack the first backup is not durable until\n// the initial sync completes and litestream uploads the initial snapshot,\n// which can take many minutes for a large replica. The platform's startup\n// probe budget (which scales with replica size) is the backstop, so\n// restoreReplica must not impose its own shorter cap and self-terminate\n// while the backup is still being produced.\n//\n// Returns the `followup` of the reservation whose backup was actually\n// restored, still open, so the caller can hand it to IncrementalSyncer's\n// first subscribe() via `followup.subscribe()`: the same replication-manager\n// that reserved the change log is the one that serves the subscription.\nasync function restoreReplica(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  changeStreamer: SnapshotReserver,\n): Promise<ReservationFollowup> {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  const progress = new RestoreProgressReporter(lc, config.replica.file);\n  try {\n    for (;;) {\n      const {reserved, followup} = await reserveAndGetSnapshotStatus(\n        lc,\n        config.taskID,\n        changeStreamer,\n      );\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = reserved);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      progress.start(reserved.replicaSize);\n      let attempt: RestoreAttempt;\n      try {\n        attempt = await tryRestore(\n          lc,\n          litestream,\n          config.replica.file,\n          reserved,\n          'view_syncer',\n          followup.signal, // abort the restore if the reservation disconnects\n        );\n      } catch (e) {\n        if (followup.signal.aborted) {\n          lc.info?.(\n            `reservation disconnected during restore. retrying`,\n            String(e),\n          );\n          continue;\n        }\n        followup.cancel();\n        throw e;\n      } finally {\n        progress.stop();\n      }\n      if (attempt.restored) {\n        result = attempt.result;\n        progress.done();\n        return followup; // connection to be converted to a subscription\n      }\n      // This attempt's reservation is stale; a fresh one is made on retry.\n      followup.cancel();\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    progress.stop();\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgEA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAUA,IAAI;CACJ,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,oBAAoB,MAAM,eAAe,IAAI,QAAQ,cAAc;CAGrE,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,SAAS,QAAQ;CAK1C,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CACN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAErE,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,MACJ,iBACF;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,MAAM,UAAU,eAAe,IAAI,QAAQ,UAAU;CAGrD,WAAW,MAAM,KAAK,WAAW,UAAU,GAAG;EAC5C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACpC;CACF;CAEA,OAAO;AACT;AAEA,SAAgB,aACd,IACA,MACA,SACA,UACA;CACA,IAAI;CACJ,MAAM,cAAc;EAClB,IAAI,CAAC,IACH,IAAI;GACF,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;EAC9C,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,mBAAmB,CAAC;EACvD;EAEF,OAAO;CACT;CAEA,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAGpD,iBAAiB,WAAW,oBAAoB;EAC9C,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,sBAAsB,IAAI,OAAO,QAAQ,CAAC;CAEzD,iBAAiB,WAAW,4BAA4B;EACtD,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,8BAA8B,IAAI,OAAO,QAAQ,CAAC;CAKjE,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,sBACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,SAAQ,GAAG,OAAsB,sBAAsB;GAC/D,IAAI,OAAO,QAAQ,UACjB,EAAE,QAAQ,KAAK,IAAI,GAAG,GAAG,IAAI,QAAQ;EAEzC,SAAS,GAAG;GACV,GAAG,OAAO,4DAA4D,CAAC;EACzE;CACF;AACF;AAEA,SAAS,8BACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,KAAK,kBAAiB,GAAG,OAG9B,sBAAsB;GACzB,IAAI,OAAO,QAAQ,YAAY,OAAO,iBAAiB,UACrD,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,IAAI,QAAQ;EAExD,SAAS,GAAG;GACV,GAAG,OACD,oEACA,CACF;EACF;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAc1B,eAAe,eACb,IACA,QACA,gBAC8B;CAC9B,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,MAAM,WAAW,IAAI,wBAAwB,IAAI,OAAO,QAAQ,IAAI;CACpE,IAAI;EACF,SAAS;GACP,MAAM,EAAC,UAAU,aAAY,MAAM,4BACjC,IACA,OAAO,QACP,cACF;GAEA,CAAC,CAAC,aAAa;GACf,MAAM,aAA+B;IAAC,GAAG,OAAO;IAAY;GAAS;GACrE,SAAS,MAAM,SAAS,WAAW;GACnC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WACd,IACA,YACA,OAAO,QAAQ,MACf,UACA,eACA,SAAS,MACX;GACF,SAAS,GAAG;IACV,IAAI,SAAS,OAAO,SAAS;KAC3B,GAAG,OACD,qDACA,OAAO,CAAC,CACV;KACA;IACF;IACA,SAAS,OAAO;IAChB,MAAM;GACR,UAAU;IACR,SAAS,KAAK;GAChB;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB,SAAS,KAAK;IACd,OAAO;GACT;GAEA,SAAS,OAAO;GAChB,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EACR,SAAS,KAAK;EAMd,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n  ChangeStreamerHttpClient,\n  type ReservationFollowup,\n  type SnapshotReserver,\n} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {\n  startReplicaBackupProcess,\n  tryRestore,\n  type RestoreAttempt,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {RestoreProgressReporter} from '../services/litestream/restore-progress.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n  ThreadWriteWorkerClient,\n  type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  installProfileHandler(parent, workerName);\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n    replicationLag: {initialReadinessThresholdMs, maxReadinessDelayMs},\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  //\n  // The reservation's connection (`initialConnection`) is threaded through to\n  // IncrementalSyncer's first subscribe(), so the replication-manager that\n  // reserved the change log for this restore is the one that serves the\n  // subscription (single-connection handoff).\n  let initialConnection: ReservationFollowup | undefined;\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    initialConnection = await restoreReplica(lc, config, changeStreamer);\n  }\n\n  const {\n    file: dbPath,\n    walMode,\n    pageSize,\n  } = await setupReplica(lc, fileMode, config.replica);\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode, pageSize);\n\n  // Create the write worker for async SQLite writes. Enable write-path\n  // checkpoint backpressure only for the backup replicator on litestream v5,\n  // which owns the replica litestream backs up.\n  const pragmas = getPragmaConfig(fileMode);\n  const checkpoint: ForceCheckpointConfig | null =\n    mode === 'backup' &&\n    config.litestream.backupURL &&\n    config.litestream.backupUsingV5 &&\n    config.litestream.forceCheckpointThresholdMB\n      ? {\n          checkpointThresholdPages:\n            (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n            pageSize,\n          maxWalPages: !config.litestream.maxWalSizeMB\n            ? undefined\n            : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n        }\n      : null;\n\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n  // The backup-replicator manages the litestream backup process. It is\n  // started after the write worker has prepared the db file in order to\n  // avoid contending on the sqlite lock.\n  const processes =\n    mode === 'backup' ? new ProcessManager(lc, parent) : undefined;\n  processes?.addSubprocess(\n    startReplicaBackupProcess(lc, config.litestream, dbPath),\n    'supporting',\n    'litestream',\n  );\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n    {\n      readinessThresholdMs: initialReadinessThresholdMs,\n      maxReadinessDelayMs,\n    },\n    initialConnection,\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  void replicator.ready().then(() => {\n    parent.send(['ready', {ready: true}]);\n  });\n\n  if (!processes) {\n    return runUntilKilled(lc, parent, replicator);\n  }\n\n  try {\n    await runUntilKilled(lc, parent, replicator);\n  } catch (err) {\n    processes.logErrorAndExit(err, workerName);\n  } finally {\n    await processes.shutdown();\n  }\n}\n\nexport function setupMetrics(\n  lc: LogContext,\n  file: string,\n  walMode: WalMode,\n  pageSize: number,\n) {\n  let db: Database | undefined;\n  const getDb = () => {\n    if (!db) {\n      try {\n        db = new Database(lc, file, {readonly: true});\n      } catch (e) {\n        lc.warn?.(`unable to open ${file} for wal metrics`, e);\n      }\n    }\n    return db;\n  };\n\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  getOrCreateGauge('replica', 'wal_active_bytes', {\n    description:\n      `The actual size of active WAL frames currently in use ` +\n      `(excluding pre-allocated or reclaimed space).`,\n    unit: 'bytes',\n  }).addCallback(observeWalActiveBytes(lc, getDb, pageSize));\n\n  getOrCreateGauge('replica', 'wal_uncheckpointed_bytes', {\n    description:\n      `The size of un-checkpointed WAL frames that have not yet ` +\n      `been backfilled into the main database.`,\n    unit: 'bytes',\n  }).addCallback(observeWalUncheckpointedBytes(lc, getDb, pageSize));\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeWalActiveBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log}] = db.pragma<{log: number}>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number') {\n        o.observe(Math.max(0, log) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.('unable to inspect wal_checkpoint for active bytes metric', e);\n    }\n  };\n}\n\nfunction observeWalUncheckpointedBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log, checkpointed}] = db.pragma<{\n        log: number;\n        checkpointed: number;\n      }>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number' && typeof checkpointed === 'number') {\n        o.observe(Math.max(0, log - checkpointed) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.(\n        'unable to inspect wal_checkpoint for uncheckpointed bytes metric',\n        e,\n      );\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// View-syncers wait indefinitely for the replication-manager to publish a\n// restorable backup. On a fresh stack the first backup is not durable until\n// the initial sync completes and litestream uploads the initial snapshot,\n// which can take many minutes for a large replica. The platform's startup\n// probe budget (which scales with replica size) is the backstop, so\n// restoreReplica must not impose its own shorter cap and self-terminate\n// while the backup is still being produced.\n//\n// Returns the `followup` of the reservation whose backup was actually\n// restored, still open, so the caller can hand it to IncrementalSyncer's\n// first subscribe() via `followup.subscribe()`: the same replication-manager\n// that reserved the change log is the one that serves the subscription.\nasync function restoreReplica(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  changeStreamer: SnapshotReserver,\n): Promise<ReservationFollowup> {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  const progress = new RestoreProgressReporter(lc, config.replica.file);\n  try {\n    for (;;) {\n      const {reserved, followup} = await reserveAndGetSnapshotStatus(\n        lc,\n        config.taskID,\n        changeStreamer,\n      );\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = reserved);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      progress.start(reserved.replicaSize);\n      let attempt: RestoreAttempt;\n      try {\n        attempt = await tryRestore(\n          lc,\n          litestream,\n          config.replica.file,\n          reserved,\n          'view_syncer',\n          followup.signal, // abort the restore if the reservation disconnects\n        );\n      } catch (e) {\n        if (followup.signal.aborted) {\n          lc.info?.(\n            `reservation disconnected during restore. retrying`,\n            String(e),\n          );\n          continue;\n        }\n        followup.cancel();\n        throw e;\n      } finally {\n        progress.stop();\n      }\n      if (attempt.restored) {\n        result = attempt.result;\n        progress.done();\n        return followup; // connection to be converted to a subscription\n      }\n      // This attempt's reservation is stale; a fresh one is made on retry.\n      followup.cancel();\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    progress.stop();\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqEA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,KAEN,gBAAgB,EAAC,6BAA6B,0BAC5C;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAUA,IAAI;CACJ,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,oBAAoB,MAAM,eAAe,IAAI,QAAQ,cAAc;CAGrE,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,SAAS,QAAQ;CAK1C,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CAEN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAKrE,MAAM,YACJ,SAAS,WAAW,IAAI,eAAe,IAAI,MAAM,IAAI,KAAA;CACvD,WAAW,cACT,0BAA0B,IAAI,OAAO,YAAY,MAAM,GACvD,cACA,YACF;CAEA,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,MACJ;EACE,sBAAsB;EACtB;CACF,GACA,iBACF;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,WAAgB,MAAM,EAAE,WAAW;EACjC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CAED,IAAI,CAAC,WACH,OAAO,eAAe,IAAI,QAAQ,UAAU;CAG9C,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,UAAU;CAC7C,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,UAAU;CAC3C,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAEA,SAAgB,aACd,IACA,MACA,SACA,UACA;CACA,IAAI;CACJ,MAAM,cAAc;EAClB,IAAI,CAAC,IACH,IAAI;GACF,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;EAC9C,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,mBAAmB,CAAC;EACvD;EAEF,OAAO;CACT;CAEA,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAGpD,iBAAiB,WAAW,oBAAoB;EAC9C,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,sBAAsB,IAAI,OAAO,QAAQ,CAAC;CAEzD,iBAAiB,WAAW,4BAA4B;EACtD,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,8BAA8B,IAAI,OAAO,QAAQ,CAAC;CAKjE,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,sBACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,SAAQ,GAAG,OAAsB,sBAAsB;GAC/D,IAAI,OAAO,QAAQ,UACjB,EAAE,QAAQ,KAAK,IAAI,GAAG,GAAG,IAAI,QAAQ;EAEzC,SAAS,GAAG;GACV,GAAG,OAAO,4DAA4D,CAAC;EACzE;CACF;AACF;AAEA,SAAS,8BACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,KAAK,kBAAiB,GAAG,OAG9B,sBAAsB;GACzB,IAAI,OAAO,QAAQ,YAAY,OAAO,iBAAiB,UACrD,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,IAAI,QAAQ;EAExD,SAAS,GAAG;GACV,GAAG,OACD,oEACA,CACF;EACF;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAc1B,eAAe,eACb,IACA,QACA,gBAC8B;CAC9B,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,MAAM,WAAW,IAAI,wBAAwB,IAAI,OAAO,QAAQ,IAAI;CACpE,IAAI;EACF,SAAS;GACP,MAAM,EAAC,UAAU,aAAY,MAAM,4BACjC,IACA,OAAO,QACP,cACF;GAEA,CAAC,CAAC,aAAa;GACf,MAAM,aAA+B;IAAC,GAAG,OAAO;IAAY;GAAS;GACrE,SAAS,MAAM,SAAS,WAAW;GACnC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WACd,IACA,YACA,OAAO,QAAQ,MACf,UACA,eACA,SAAS,MACX;GACF,SAAS,GAAG;IACV,IAAI,SAAS,OAAO,SAAS;KAC3B,GAAG,OACD,qDACA,OAAO,CAAC,CACV;KACA;IACF;IACA,SAAS,OAAO;IAChB,MAAM;GACR,UAAU;IACR,SAAS,KAAK;GAChB;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB,SAAS,KAAK;IACd,OAAO;GACT;GAEA,SAAS,OAAO;GAChB,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EACR,SAAS,KAAK;EAMd,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/services/change-source/pg/change-source-init.js

@@ -206,7 +206,7 @@ async function getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailov
 					const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);
 					lc.info?.(`replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`, { replica });
 					if (inactiveMs >= gracePeriodMs) {
-						const reserved = await claimSlotForResumption(lc, sql, shard, "resume-replica", replica.slot);
+						const reserved = await claimSlotForResumption(lc, sql, shard, "resume-replica", replica);
 						if (!reserved) lc.warn?.(`unable to resume replica ${replica.id}@${replica.slot}`);
 						else {
 							lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);

package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport type {ReplicaConstraints} from '../../litestream/commands.ts';\nimport {\n  getSubscriptionStateAndContext,\n  type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n  restoreReplica,\n  type ConstrainingPurgeLock,\n  type InitializeResult,\n  type RestoreOptions,\n  type ResumableBackup,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n  initialSync,\n  type InitialSyncOptions,\n  type ReplicaOptions,\n  type ServerContext,\n} from './initial-sync.ts';\nimport {toBigInt, toStateVersionString, type LSN} from './lsn.ts';\nimport {\n  claimSlotForResumption,\n  createReplicaAndSlot,\n  type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n  dropShard,\n  getActiveReplicas,\n  getReplicaAtVersion,\n  getRestoreCandidates,\n  initRestoreReplica,\n  internalPublicationPrefix,\n  type ReplicaState,\n} from './schema/shard.ts';\n\nexport type InitializeOptions = ReplicaOptions & {\n  // Create a new slot for the replica rather than taking over\n  // an existing one (i.e. high-availability mode).\n  slotPerReplica: boolean | undefined;\n\n  inactiveReplicaGracePeriodMs: number;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n  lagReportIntervalMs = 0,\n  restoreOptions: RestoreOptions = {},\n  {\n    epoch,\n    slotPerReplica,\n    backupV5,\n    inactiveReplicaGracePeriodMs,\n  }: InitializeOptions = {\n    epoch: 0,\n    slotPerReplica: false,\n    backupV5: true,\n    inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n  },\n  streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n  const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n  try {\n    await ensureShardSchema(\n      lc,\n      db,\n      shard,\n      syncOptions.installPartialIndexTriggers,\n    );\n\n    const {\n      replica: restoredReplica,\n      purgeLock,\n      pgChangeLogBehindSlot,\n      resumableBackup,\n    } = slotPerReplica\n      ? await forkOrResumeReplica(\n          lc,\n          db,\n          shard,\n          epoch,\n          syncOptions.replicationSlotFailover ?? false,\n          replicaDbFile,\n          restoreOptions,\n          inactiveReplicaGracePeriodMs,\n        )\n      : await selectAndRestoreReplica(\n          lc,\n          db,\n          shard,\n          replicaDbFile,\n          restoreOptions,\n        );\n\n    let initialSyncedReplica: ReplicaState | undefined;\n    await initReplica(\n      lc,\n      `replica-${shard.appID}-${shard.shardNum}`,\n      replicaDbFile,\n      async (log, tx) => {\n        // Falling back to initial sync abandons the restore, so release the\n        // resources acquired for it, e.g. a replication slot claimed (or\n        // created) for a resumed (or forked) replica, which would otherwise\n        // remain active indefinitely.\n        await restoreOptions.cleanup?.release();\n        // The purge lock on the change-db must be released before performing\n        // initial sync; if the change-db and upstream are the same db, a lock-holding\n        // transaction will prevent a replication slot from being created. This awkward\n        // dependency can go away with RMv2. (The purge lock is also released by\n        // the cleanup when it is registered with it; releasing it is idempotent.)\n        await purgeLock?.release();\n        initialSyncedReplica = await initialSync(\n          log,\n          shard,\n          tx,\n          upstreamURI,\n          syncOptions,\n          context,\n          {epoch, backupV5},\n          restoreOptions.cleanup,\n        );\n      },\n    );\n\n    const replica = new Database(lc, replicaDbFile);\n    const subscriptionState = getSubscriptionStateAndContext(\n      new StatementRunner(replica),\n    );\n    replica.close();\n\n    // Check that upstream is properly setup, and throw an AutoReset to re-run\n    // initial sync if not.\n    const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n      lc,\n      db,\n      shard,\n      subscriptionState,\n      (initialSyncedReplica ?? restoredReplica)?.id,\n    );\n\n    // With litestream v5, every replication-manager backs up to its own\n    // backup lineage.\n    const newBackupLineage = backupV5 && !resumableBackup;\n    const backupPath = initialSyncedReplica\n      ? // If initial sync was performed, use that initial backupPath.\n        initialSyncedReplica.backupPath\n      : // Otherwise, use the lineage prepared by the restore, or a new, unique\n        // path when backing up with litestream v5. This will be recorded in the\n        // replicas table by the PostgresChangeSource.\n        backupV5\n        ? (resumableBackup?.backupPath ?? String(Date.now()))\n        : (restoredReplica?.backupPath ?? null);\n\n    const changeSource = new PostgresChangeSource(\n      lc,\n      upstreamURI,\n      shard,\n      upstreamReplica,\n      pgVersion,\n      {backupPath, backupV5},\n      context,\n      lagReportIntervalMs,\n      syncOptions.textCopy,\n      streamInboundTimeoutMs,\n    );\n\n    const destinationBackupURL =\n      backupPath && restoreOptions.litestream?.backupURL\n        ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n        : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n          restoreOptions.litestream?.backupURL;\n\n    return {\n      subscriptionState,\n      changeSource,\n      destinationBackupURL,\n      // The replica this change stream belongs to. It is part of the identity\n      // the SQLite change log records, because a generation (i.e.\n      // `replicaVersion`) is shared by every sibling of a forked replica and so\n      // cannot distinguish two siblings' logs.\n      replicaID: upstreamReplica.id,\n      waitForBackupBeforeServing:\n        // Wait for the first backup if there was an initial sync,\n        initialSyncedReplica !== undefined ||\n        // or if the destination differs from where it was restored\n        // (i.e. backupV5).\n        backupPath !== (restoredReplica?.backupPath ?? null),\n      pgChangeLogBehindSlot,\n      newBackupLineage,\n      walKeeper: resumableBackup?.walKeeper,\n    };\n  } finally {\n    await db.end();\n  }\n}\n\ntype RestoredReplica = {\n  replica: ReplicaState | undefined;\n  purgeLock: ConstrainingPurgeLock | null;\n  pgChangeLogBehindSlot?: boolean | undefined;\n  resumableBackup?: ResumableBackup | undefined;\n};\n\n// RMv1: Selects a replica to restore from and returns it, with the\n//       intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaFile: string,\n  {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n): Promise<RestoredReplica> {\n  // The purge lock constrains the generation of the replica to restore. With\n  // a shared replication slot, the change-log is resumed from its head\n  // without being checked against the slot's position: the slot is still\n  // being acked by the previous replication-manager, on an idle upstream past\n  // the change-log's head (see PgChangeLogPurgeLocker).\n  const lock = await acquirePurgeLock?.();\n  assert(lock !== 'behind-slot', 'unexpected purge-lock result');\n  const purgeLock = lock ?? null;\n  const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n    // filter to the generation specified by the purge lock, if present\n    ({generation}) => generation === (purgeLock?.replicaVersion ?? generation),\n  );\n  if (replicas.length === 0) {\n    lc.info?.(`no suitable replicas to restore from`, {replicas});\n    return {replica: undefined, purgeLock};\n  }\n  const [replica] = replicas;\n\n  if (litestream?.backupURL) {\n    const constraints = restoreConstraints(purgeLock, replica);\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = replica;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {replicas},\n    );\n    const resumableBackup = await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n      litestream.forkBackup\n        ? {type: 'fork', baseURL: backupBaseURL}\n        : undefined,\n      cleanup,\n    );\n    return {replica, purgeLock, resumableBackup};\n  }\n  return {replica, purgeLock};\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  replicaFile: string,\n  {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n  gracePeriodMs: number,\n): Promise<RestoredReplica> {\n  const result = await getSourceAndDestinationReplicas(\n    lc,\n    sql,\n    shard,\n    epoch,\n    slotFailover,\n    gracePeriodMs,\n  );\n  if (!result) {\n    // can't restore, must initial-sync\n    return {replica: undefined, purgeLock: null};\n  }\n  const {restoreFrom, replicateTo, slotLSN, slotSession} = result;\n  // The slot is held active by a session in this process until it is taken\n  // over by the PostgresChangeSource. If initialization fails before then,\n  // the session must be released, or the slot remains active indefinitely,\n  // preventing the replica from being resumed (or forked) by a retry.\n  cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () =>\n    slotSession.destroy(),\n  );\n\n  // Acquired only after the replication slot has been created (or claimed),\n  // and only if the change-log can be resumed from the slot's position.\n  //\n  // For a fork, this is the case whenever the change-log is maintained (i.e.\n  // by a replication-manager that acks the slot only once a transaction has\n  // been stored in the change-log): the fork waits for the source replica's\n  // slot to pass the position after the creation of the new replica, which is\n  // a transaction (in the publication) after the new slot's position.\n  //\n  // For a resumed slot, the slot can also have been acked past the head of a\n  // maintained change-log, for changes outside of the publication (e.g. on an\n  // idle upstream). The change-log is then re-initialized unnecessarily, which\n  // resets subscribers behind the replica. (This could be avoided by recording\n  // the positions acked for such changes in the change-log, e.g. an\n  // `ackedThrough` column, and checking the slot's position against that.\n  // Resumption is uncommon, however, and involves downtime anyway.)\n  const lock = acquirePurgeLock\n    ? await acquirePurgeLock(toStateVersionString(slotLSN))\n    : null;\n  const pgChangeLogBehindSlot = lock === 'behind-slot';\n  const purgeLock = lock === 'behind-slot' ? null : lock;\n  // Absent a purge lock, still constrain the restore to the source's\n  // generation, so that a local replica left behind by a different\n  // generation (e.g. a failed initial sync) is not mistaken for it.\n  const constraints = restoreConstraints(purgeLock, restoreFrom);\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {restoreFrom, replicateTo},\n    );\n    const resumableBackup = await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n      litestream.forkBackup\n        ? {type: 'fork', baseURL: backupBaseURL}\n        : undefined,\n      cleanup,\n    );\n    return {\n      replica: replicateTo,\n      purgeLock,\n      pgChangeLogBehindSlot,\n      resumableBackup,\n    };\n  }\n  return {replica: replicateTo, purgeLock, pgChangeLogBehindSlot};\n}\n\nfunction restoreConstraints(\n  purgeLock: ConstrainingPurgeLock | null,\n  {generation}: ReplicaState,\n): ReplicaConstraints {\n  return purgeLock ?? {replicaVersion: generation, minWatermark: ''};\n}\n\nasync function currentWalLSN(sql: PostgresDB): Promise<LSN> {\n  const [{lsn}] = await sql<{lsn: LSN}[]>`SELECT pg_current_wal_lsn() AS lsn`;\n  return lsn;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 5_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  gracePeriodMs: number,\n  pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<\n  | {\n      restoreFrom: ReplicaState;\n      replicateTo: ReplicaState;\n      /** The position of the slot from which replication resumes. */\n      slotLSN: LSN;\n      /**\n       * The session holding the destination slot active until it is taken\n       * over by the PostgresChangeSource. Destroying it releases the slot.\n       */\n      slotSession: Readable;\n    }\n  | undefined\n> {\n  const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n  let destination: ReplicationSlotResult<void> | undefined;\n  // The position after the creation of the destination replica.\n  let forkLSN: LSN | undefined;\n\n  try {\n    for (let i = 0; ; i++) {\n      if (i > 0) {\n        await sleep(pollIntervalMs);\n      }\n      const replicas = await getRestoreCandidates(lc, sql, shard, epoch);\n      if (replicas.length === 0) {\n        lc.info?.(`no suitable replicas to restore from`, {replicas});\n        destination?.initialSession.destroy();\n        return undefined;\n      }\n      for (const replica of replicas) {\n        // Track inactivity to resume replicas after a grace period.\n        if (replica.active) {\n          inactiveSince.delete(replica.id);\n        } else if (!inactiveSince.has(replica.id)) {\n          inactiveSince.set(replica.id, Date.now());\n        }\n\n        // Note: Only `active` InitialSync replicas are returned from\n        // getRestoreCandidates().\n        if (replica.stage === InitialSync) {\n          // Log periodically; initial-sync can be long\n          if (i % 12 === 0) {\n            lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n          }\n          break;\n        }\n\n        if (replica.stage === Replicate) {\n          const {active, confirmedFlushLsn} = replica;\n          if (active) {\n            // Create a replication slot to fork the active replica.\n            destination ??= await createReplicaAndSlot(\n              lc,\n              sql,\n              'fork-replica-session',\n              shard,\n              epoch,\n              Date.now().toString(), // replicaID\n              slotFailover,\n              {\n                backupPath: null, // set only after the backup has been confirmed\n                backupV5: true, // RMv2 requires backupV5\n              },\n              () => promiseVoid,\n              ReplicaStage.Restore,\n            );\n            // The replica row of the destination is inserted in a transaction\n            // (in the publication) after the destination slot's position.\n            // Waiting for the source replica's slot to pass it (rather than\n            // just the destination slot's position) ensures that a source\n            // replica that maintains the PG change-log has stored a\n            // transaction past the destination slot's position, i.e. that the\n            // change-log can be resumed from the destination slot.\n            forkLSN ??= await currentWalLSN(sql);\n            if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {\n              lc.info?.(\n                `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`,\n                {replica},\n              );\n              break;\n            }\n          }\n\n          // If the replica is past the destination LSN, fork it, regardless of\n          // whether its slot is still active. Forking is preferable to resuming,\n          // as resuming removes the replica from being a candidate for\n          // subsequent forks, and carries the risk of stealing the slot from a\n          // task attempting to reconnect.\n          if (\n            destination &&\n            forkLSN &&\n            toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)\n          ) {\n            lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n              replica,\n            });\n            const replicateTo = must(\n              await initRestoreReplica(sql, shard, {\n                sourceID: replica.id,\n                destID: destination.replica.id,\n              }),\n              `replica ${destination.replica.id} was deleted`,\n            );\n            return {\n              restoreFrom: replica,\n              replicateTo,\n              slotLSN: destination.slot.consistent_point,\n              slotSession: destination.initialSession,\n            };\n          }\n        }\n\n        if (\n          replica.stage === ReplicaStage.Restore &&\n          replica.active &&\n          replica.id !== destination?.replica.id\n        ) {\n          // Another task is restoring (e.g. resuming) this replica.\n          if (i % 12 === 0) {\n            lc.info?.(\n              `waiting for replica ${replica.id}@${replica.slot} being restored by another task`,\n              {replica},\n            );\n          }\n        }\n\n        if (replica.backupPath && !replica.active) {\n          // An inactive replica that has a backupPath may be orphaned, or it\n          // may be an active task that was temporarily disconnected and\n          // attempting to reestablish a session.\n          //\n          // Resume the replica after a grace period to avoid stealing the slot\n          // from an active task that was temporarily disconnected. Resumption\n          // is a last resort only taken in the absence of alternatives.\n          const now = Date.now();\n          const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n          lc.info?.(\n            `replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`,\n            {replica},\n          );\n          if (inactiveMs >= gracePeriodMs) {\n            const reserved = await claimSlotForResumption(\n              lc,\n              sql,\n              shard,\n              'resume-replica',\n              replica.slot,\n            );\n            if (!reserved) {\n              lc.warn?.(\n                `unable to resume replica ${replica.id}@${replica.slot}`,\n              );\n            } else {\n              lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n              // If a new replication slot was created in anticipation of forking,\n              // cancel it\n              destination?.initialSession.destroy();\n              return {\n                restoreFrom: replica,\n                replicateTo: reserved.replica,\n                // The slot is inactive, so its position is fixed.\n                slotLSN: replica.confirmedFlushLsn,\n                slotSession: reserved.reservation,\n              };\n            }\n          }\n        }\n      }\n    }\n  } catch (e) {\n    destination?.initialSession.destroy();\n    throw e;\n  }\n}\n\nasync function checkAndUpdateUpstream(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  {\n    replicaVersion,\n    publications: subscribed,\n    initialSyncContext,\n  }: SubscriptionStateAndContext,\n  replicaID: string | undefined,\n) {\n  const upstreamReplica = await getReplicaAtVersion(\n    lc,\n    sql,\n    shard,\n    replicaVersion,\n    replicaID,\n    initialSyncContext,\n  );\n  if (!upstreamReplica) {\n    throw new AutoResetSignal(\n      `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n    );\n  }\n\n  // Verify that the publications match what is being replicated.\n  const requested = shard.publications.toSorted();\n  const replicated = upstreamReplica.publications\n    .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n    .sort();\n  if (!deepEqual(requested, replicated)) {\n    lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n    await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n    throw new AutoResetSignal(\n      `Requested publications [${requested}] do not match configured ` +\n        `publications: [${replicated}]`,\n    );\n  }\n\n  // Sanity check: The subscription state on the replica should have the\n  // same publications. This should be guaranteed by the equivalence of the\n  // replicaVersion, but it doesn't hurt to verify.\n  if (!deepEqual(upstreamReplica.publications, subscribed)) {\n    throw new AutoResetSignal(\n      `Upstream publications [${upstreamReplica.publications}] do not ` +\n        `match subscribed publications [${subscribed}]`,\n    );\n  }\n\n  // Verify that the publications exist.\n  const exists = await sql`\n    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n  `.values();\n  if (exists.length !== subscribed.length) {\n    throw new AutoResetSignal(\n      `Upstream publications [${exists.flat()}] do not contain ` +\n        `all subscribed publications [${subscribed}]`,\n    );\n  }\n\n  const {slot} = upstreamReplica;\n  const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n  /*sql*/ `\n    SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n      WHERE slot_name = ${slot}`;\n  if (result.length === 0) {\n    throw new AutoResetSignal(`replication slot ${slot} is missing`);\n  }\n  const [{restartLSN, walStatus}] = result;\n  if (restartLSN === null || walStatus === 'lost') {\n    throw new AutoResetSignal(\n      `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n    );\n  }\n  const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n    SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n  return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAgEA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,iCACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;AAChC,GACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,MAAM,EACJ,SAAS,iBACT,WACA,uBACA,oBACE,iBACA,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,MAAM,eAAe,SAAS,QAAQ;GAMtC,MAAM,WAAW,QAAQ;GACzB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;GAAQ,GAChB,eAAe,OACjB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAIA,MAAM,mBAAmB,YAAY,CAAC;EACtC,MAAM,aAAa,uBAEf,qBAAqB,aAIrB,WACG,iBAAiB,cAAc,OAAO,KAAK,IAAI,CAAC,IAChD,iBAAiB,cAAc;EAqBtC,OAAO;GACL;GACA,cAAA,IArBuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;GAAQ,GACrB,SACA,qBACA,YAAY,UACZ,sBAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;GACjD;GACA;GACA,WAAW,iBAAiB;EAC9B;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAWA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,kBAAkB,WACL;CAM1B,MAAM,OAAO,MAAM,mBAAmB;CACtC,OAAO,SAAS,eAAe,8BAA8B;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBAAgB,gBAAgB,WAAW,kBAAkB,WACjE;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D,OAAO;GAAC,SAAS,KAAA;GAAW;EAAS;CACvC;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,cAAc,mBAAmB,WAAW,OAAO;EACzD,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EAWA,OAAO;GAAC;GAAS;GAAW,iBAAA,MAVE,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,GACJ,OACF;EAC2C;CAC7C;CACA,OAAO;EAAC;EAAS;CAAS;AAC5B;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,kBAAkB,WAC/B,eAC0B;CAC1B,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QAEH,OAAO;EAAC,SAAS,KAAA;EAAW,WAAW;CAAI;CAE7C,MAAM,EAAC,aAAa,aAAa,SAAS,gBAAe;CAKzD,SAAS,UAAU,oBAAoB,YAAY,cACjD,YAAY,QAAQ,CACtB;CAkBA,MAAM,OAAO,mBACT,MAAM,iBAAiB,qBAAqB,OAAO,CAAC,IACpD;CACJ,MAAM,wBAAwB,SAAS;CACvC,MAAM,YAAY,SAAS,gBAAgB,OAAO;CAIlD,MAAM,cAAc,mBAAmB,WAAW,WAAW;CAE7D,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EAWA,OAAO;GACL,SAAS;GACT;GACA;GACA,iBAAA,MAd4B,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,GACJ,OACF;EAMA;CACF;CACA,OAAO;EAAC,SAAS;EAAa;EAAW;CAAqB;AAChE;AAEA,SAAS,mBACP,WACA,EAAC,cACmB;CACpB,OAAO,aAAa;EAAC,gBAAgB;EAAY,cAAc;CAAE;AACnE;AAEA,eAAe,cAAc,KAA+B;CAC1D,MAAM,CAAC,EAAC,SAAQ,MAAM,GAAiB;CACvC,OAAO;AACT;AAEA,IAAM,2BAA2B;AACjC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BAcjB;CACA,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;CAEJ,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,IAAI,KAAK,OAAO,KAAK;GACjE,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,aAAa,eAAe,QAAQ;IACpC;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IAAI,IAAI,OAAO,GACb,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;MACZ,SACM,aACN,CACF;MAQA,YAAY,MAAM,cAAc,GAAG;MACnC,IAAI,SAAS,iBAAiB,IAAI,SAAS,OAAO,GAAG;OACnD,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,QAAQ,UAAU,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,iBAAiB,IAC7I,EAAC,QAAO,CACV;OACA;MACF;KACF;KAOA,IACE,eACA,WACA,SAAS,iBAAiB,KAAK,SAAS,OAAO,GAC/C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OACL,aAAa;OACb,aATkB,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAIlC;OACA,SAAS,YAAY,KAAK;OAC1B,aAAa,YAAY;MAC3B;KACF;IACF;IAEA,IACE,QAAQ,UAAU,KAClB,QAAQ,UACR,QAAQ,OAAO,aAAa,QAAQ;SAGhC,IAAI,OAAO,GACb,GAAG,OACD,uBAAuB,QAAQ,GAAG,GAAG,QAAQ,KAAK,kCAClD,EAAC,QAAO,CACV;IAAA;IAIJ,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,wBAAwB,WAAW,KACzE,EAAC,QAAO,CACV;KACA,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,QAAQ,IACV;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,aAAa,eAAe,QAAQ;OACpC,OAAO;QACL,aAAa;QACb,aAAa,SAAS;QAEtB,SAAS,QAAQ;QACjB,aAAa,SAAS;OACxB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,aAAa,eAAe,QAAQ;EACpC,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}
+{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport type {ReplicaConstraints} from '../../litestream/commands.ts';\nimport {\n  getSubscriptionStateAndContext,\n  type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n  restoreReplica,\n  type ConstrainingPurgeLock,\n  type InitializeResult,\n  type RestoreOptions,\n  type ResumableBackup,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n  initialSync,\n  type InitialSyncOptions,\n  type ReplicaOptions,\n  type ServerContext,\n} from './initial-sync.ts';\nimport {toBigInt, toStateVersionString, type LSN} from './lsn.ts';\nimport {\n  claimSlotForResumption,\n  createReplicaAndSlot,\n  type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n  dropShard,\n  getActiveReplicas,\n  getReplicaAtVersion,\n  getRestoreCandidates,\n  initRestoreReplica,\n  internalPublicationPrefix,\n  type ReplicaState,\n} from './schema/shard.ts';\n\nexport type InitializeOptions = ReplicaOptions & {\n  // Create a new slot for the replica rather than taking over\n  // an existing one (i.e. high-availability mode).\n  slotPerReplica: boolean | undefined;\n\n  inactiveReplicaGracePeriodMs: number;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n  lagReportIntervalMs = 0,\n  restoreOptions: RestoreOptions = {},\n  {\n    epoch,\n    slotPerReplica,\n    backupV5,\n    inactiveReplicaGracePeriodMs,\n  }: InitializeOptions = {\n    epoch: 0,\n    slotPerReplica: false,\n    backupV5: true,\n    inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n  },\n  streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n  const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n  try {\n    await ensureShardSchema(\n      lc,\n      db,\n      shard,\n      syncOptions.installPartialIndexTriggers,\n    );\n\n    const {\n      replica: restoredReplica,\n      purgeLock,\n      pgChangeLogBehindSlot,\n      resumableBackup,\n    } = slotPerReplica\n      ? await forkOrResumeReplica(\n          lc,\n          db,\n          shard,\n          epoch,\n          syncOptions.replicationSlotFailover ?? false,\n          replicaDbFile,\n          restoreOptions,\n          inactiveReplicaGracePeriodMs,\n        )\n      : await selectAndRestoreReplica(\n          lc,\n          db,\n          shard,\n          replicaDbFile,\n          restoreOptions,\n        );\n\n    let initialSyncedReplica: ReplicaState | undefined;\n    await initReplica(\n      lc,\n      `replica-${shard.appID}-${shard.shardNum}`,\n      replicaDbFile,\n      async (log, tx) => {\n        // Falling back to initial sync abandons the restore, so release the\n        // resources acquired for it, e.g. a replication slot claimed (or\n        // created) for a resumed (or forked) replica, which would otherwise\n        // remain active indefinitely.\n        await restoreOptions.cleanup?.release();\n        // The purge lock on the change-db must be released before performing\n        // initial sync; if the change-db and upstream are the same db, a lock-holding\n        // transaction will prevent a replication slot from being created. This awkward\n        // dependency can go away with RMv2. (The purge lock is also released by\n        // the cleanup when it is registered with it; releasing it is idempotent.)\n        await purgeLock?.release();\n        initialSyncedReplica = await initialSync(\n          log,\n          shard,\n          tx,\n          upstreamURI,\n          syncOptions,\n          context,\n          {epoch, backupV5},\n          restoreOptions.cleanup,\n        );\n      },\n    );\n\n    const replica = new Database(lc, replicaDbFile);\n    const subscriptionState = getSubscriptionStateAndContext(\n      new StatementRunner(replica),\n    );\n    replica.close();\n\n    // Check that upstream is properly setup, and throw an AutoReset to re-run\n    // initial sync if not.\n    const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n      lc,\n      db,\n      shard,\n      subscriptionState,\n      (initialSyncedReplica ?? restoredReplica)?.id,\n    );\n\n    // With litestream v5, every replication-manager backs up to its own\n    // backup lineage.\n    const newBackupLineage = backupV5 && !resumableBackup;\n    const backupPath = initialSyncedReplica\n      ? // If initial sync was performed, use that initial backupPath.\n        initialSyncedReplica.backupPath\n      : // Otherwise, use the lineage prepared by the restore, or a new, unique\n        // path when backing up with litestream v5. This will be recorded in the\n        // replicas table by the PostgresChangeSource.\n        backupV5\n        ? (resumableBackup?.backupPath ?? String(Date.now()))\n        : (restoredReplica?.backupPath ?? null);\n\n    const changeSource = new PostgresChangeSource(\n      lc,\n      upstreamURI,\n      shard,\n      upstreamReplica,\n      pgVersion,\n      {backupPath, backupV5},\n      context,\n      lagReportIntervalMs,\n      syncOptions.textCopy,\n      streamInboundTimeoutMs,\n    );\n\n    const destinationBackupURL =\n      backupPath && restoreOptions.litestream?.backupURL\n        ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n        : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n          restoreOptions.litestream?.backupURL;\n\n    return {\n      subscriptionState,\n      changeSource,\n      destinationBackupURL,\n      // The replica this change stream belongs to. It is part of the identity\n      // the SQLite change log records, because a generation (i.e.\n      // `replicaVersion`) is shared by every sibling of a forked replica and so\n      // cannot distinguish two siblings' logs.\n      replicaID: upstreamReplica.id,\n      waitForBackupBeforeServing:\n        // Wait for the first backup if there was an initial sync,\n        initialSyncedReplica !== undefined ||\n        // or if the destination differs from where it was restored\n        // (i.e. backupV5).\n        backupPath !== (restoredReplica?.backupPath ?? null),\n      pgChangeLogBehindSlot,\n      newBackupLineage,\n      walKeeper: resumableBackup?.walKeeper,\n    };\n  } finally {\n    await db.end();\n  }\n}\n\ntype RestoredReplica = {\n  replica: ReplicaState | undefined;\n  purgeLock: ConstrainingPurgeLock | null;\n  pgChangeLogBehindSlot?: boolean | undefined;\n  resumableBackup?: ResumableBackup | undefined;\n};\n\n// RMv1: Selects a replica to restore from and returns it, with the\n//       intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaFile: string,\n  {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n): Promise<RestoredReplica> {\n  // The purge lock constrains the generation of the replica to restore. With\n  // a shared replication slot, the change-log is resumed from its head\n  // without being checked against the slot's position: the slot is still\n  // being acked by the previous replication-manager, on an idle upstream past\n  // the change-log's head (see PgChangeLogPurgeLocker).\n  const lock = await acquirePurgeLock?.();\n  assert(lock !== 'behind-slot', 'unexpected purge-lock result');\n  const purgeLock = lock ?? null;\n  const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n    // filter to the generation specified by the purge lock, if present\n    ({generation}) => generation === (purgeLock?.replicaVersion ?? generation),\n  );\n  if (replicas.length === 0) {\n    lc.info?.(`no suitable replicas to restore from`, {replicas});\n    return {replica: undefined, purgeLock};\n  }\n  const [replica] = replicas;\n\n  if (litestream?.backupURL) {\n    const constraints = restoreConstraints(purgeLock, replica);\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = replica;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {replicas},\n    );\n    const resumableBackup = await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n      litestream.forkBackup\n        ? {type: 'fork', baseURL: backupBaseURL}\n        : undefined,\n      cleanup,\n    );\n    return {replica, purgeLock, resumableBackup};\n  }\n  return {replica, purgeLock};\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  replicaFile: string,\n  {litestream, acquirePurgeLock, cleanup}: RestoreOptions,\n  gracePeriodMs: number,\n): Promise<RestoredReplica> {\n  const result = await getSourceAndDestinationReplicas(\n    lc,\n    sql,\n    shard,\n    epoch,\n    slotFailover,\n    gracePeriodMs,\n  );\n  if (!result) {\n    // can't restore, must initial-sync\n    return {replica: undefined, purgeLock: null};\n  }\n  const {restoreFrom, replicateTo, slotLSN, slotSession} = result;\n  // The slot is held active by a session in this process until it is taken\n  // over by the PostgresChangeSource. If initialization fails before then,\n  // the session must be released, or the slot remains active indefinitely,\n  // preventing the replica from being resumed (or forked) by a retry.\n  cleanup?.onFailure(`replication slot ${replicateTo.slot}`, () =>\n    slotSession.destroy(),\n  );\n\n  // Acquired only after the replication slot has been created (or claimed),\n  // and only if the change-log can be resumed from the slot's position.\n  //\n  // For a fork, this is the case whenever the change-log is maintained (i.e.\n  // by a replication-manager that acks the slot only once a transaction has\n  // been stored in the change-log): the fork waits for the source replica's\n  // slot to pass the position after the creation of the new replica, which is\n  // a transaction (in the publication) after the new slot's position.\n  //\n  // For a resumed slot, the slot can also have been acked past the head of a\n  // maintained change-log, for changes outside of the publication (e.g. on an\n  // idle upstream). The change-log is then re-initialized unnecessarily, which\n  // resets subscribers behind the replica. (This could be avoided by recording\n  // the positions acked for such changes in the change-log, e.g. an\n  // `ackedThrough` column, and checking the slot's position against that.\n  // Resumption is uncommon, however, and involves downtime anyway.)\n  const lock = acquirePurgeLock\n    ? await acquirePurgeLock(toStateVersionString(slotLSN))\n    : null;\n  const pgChangeLogBehindSlot = lock === 'behind-slot';\n  const purgeLock = lock === 'behind-slot' ? null : lock;\n  // Absent a purge lock, still constrain the restore to the source's\n  // generation, so that a local replica left behind by a different\n  // generation (e.g. a failed initial sync) is not mistaken for it.\n  const constraints = restoreConstraints(purgeLock, restoreFrom);\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {restoreFrom, replicateTo},\n    );\n    const resumableBackup = await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n      litestream.forkBackup\n        ? {type: 'fork', baseURL: backupBaseURL}\n        : undefined,\n      cleanup,\n    );\n    return {\n      replica: replicateTo,\n      purgeLock,\n      pgChangeLogBehindSlot,\n      resumableBackup,\n    };\n  }\n  return {replica: replicateTo, purgeLock, pgChangeLogBehindSlot};\n}\n\nfunction restoreConstraints(\n  purgeLock: ConstrainingPurgeLock | null,\n  {generation}: ReplicaState,\n): ReplicaConstraints {\n  return purgeLock ?? {replicaVersion: generation, minWatermark: ''};\n}\n\nasync function currentWalLSN(sql: PostgresDB): Promise<LSN> {\n  const [{lsn}] = await sql<{lsn: LSN}[]>`SELECT pg_current_wal_lsn() AS lsn`;\n  return lsn;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 5_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  gracePeriodMs: number,\n  pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<\n  | {\n      restoreFrom: ReplicaState;\n      replicateTo: ReplicaState;\n      /** The position of the slot from which replication resumes. */\n      slotLSN: LSN;\n      /**\n       * The session holding the destination slot active until it is taken\n       * over by the PostgresChangeSource. Destroying it releases the slot.\n       */\n      slotSession: Readable;\n    }\n  | undefined\n> {\n  const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n  let destination: ReplicationSlotResult<void> | undefined;\n  // The position after the creation of the destination replica.\n  let forkLSN: LSN | undefined;\n\n  try {\n    for (let i = 0; ; i++) {\n      if (i > 0) {\n        await sleep(pollIntervalMs);\n      }\n      const replicas = await getRestoreCandidates(lc, sql, shard, epoch);\n      if (replicas.length === 0) {\n        lc.info?.(`no suitable replicas to restore from`, {replicas});\n        destination?.initialSession.destroy();\n        return undefined;\n      }\n      for (const replica of replicas) {\n        // Track inactivity to resume replicas after a grace period.\n        if (replica.active) {\n          inactiveSince.delete(replica.id);\n        } else if (!inactiveSince.has(replica.id)) {\n          inactiveSince.set(replica.id, Date.now());\n        }\n\n        // Note: Only `active` InitialSync replicas are returned from\n        // getRestoreCandidates().\n        if (replica.stage === InitialSync) {\n          // Log periodically; initial-sync can be long\n          if (i % 12 === 0) {\n            lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n          }\n          break;\n        }\n\n        if (replica.stage === Replicate) {\n          const {active, confirmedFlushLsn} = replica;\n          if (active) {\n            // Create a replication slot to fork the active replica.\n            destination ??= await createReplicaAndSlot(\n              lc,\n              sql,\n              'fork-replica-session',\n              shard,\n              epoch,\n              Date.now().toString(), // replicaID\n              slotFailover,\n              {\n                backupPath: null, // set only after the backup has been confirmed\n                backupV5: true, // RMv2 requires backupV5\n              },\n              () => promiseVoid,\n              ReplicaStage.Restore,\n            );\n            // The replica row of the destination is inserted in a transaction\n            // (in the publication) after the destination slot's position.\n            // Waiting for the source replica's slot to pass it (rather than\n            // just the destination slot's position) ensures that a source\n            // replica that maintains the PG change-log has stored a\n            // transaction past the destination slot's position, i.e. that the\n            // change-log can be resumed from the destination slot.\n            forkLSN ??= await currentWalLSN(sql);\n            if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {\n              lc.info?.(\n                `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`,\n                {replica},\n              );\n              break;\n            }\n          }\n\n          // If the replica is past the destination LSN, fork it, regardless of\n          // whether its slot is still active. Forking is preferable to resuming,\n          // as resuming removes the replica from being a candidate for\n          // subsequent forks, and carries the risk of stealing the slot from a\n          // task attempting to reconnect.\n          if (\n            destination &&\n            forkLSN &&\n            toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)\n          ) {\n            lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n              replica,\n            });\n            const replicateTo = must(\n              await initRestoreReplica(sql, shard, {\n                sourceID: replica.id,\n                destID: destination.replica.id,\n              }),\n              `replica ${destination.replica.id} was deleted`,\n            );\n            return {\n              restoreFrom: replica,\n              replicateTo,\n              slotLSN: destination.slot.consistent_point,\n              slotSession: destination.initialSession,\n            };\n          }\n        }\n\n        if (\n          replica.stage === ReplicaStage.Restore &&\n          replica.active &&\n          replica.id !== destination?.replica.id\n        ) {\n          // Another task is restoring (e.g. resuming) this replica.\n          if (i % 12 === 0) {\n            lc.info?.(\n              `waiting for replica ${replica.id}@${replica.slot} being restored by another task`,\n              {replica},\n            );\n          }\n        }\n\n        if (replica.backupPath && !replica.active) {\n          // An inactive replica that has a backupPath may be orphaned, or it\n          // may be an active task that was temporarily disconnected and\n          // attempting to reestablish a session.\n          //\n          // Resume the replica after a grace period to avoid stealing the slot\n          // from an active task that was temporarily disconnected. Resumption\n          // is a last resort only taken in the absence of alternatives.\n          const now = Date.now();\n          const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n          lc.info?.(\n            `replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`,\n            {replica},\n          );\n          if (inactiveMs >= gracePeriodMs) {\n            const reserved = await claimSlotForResumption(\n              lc,\n              sql,\n              shard,\n              'resume-replica',\n              replica,\n            );\n            if (!reserved) {\n              lc.warn?.(\n                `unable to resume replica ${replica.id}@${replica.slot}`,\n              );\n            } else {\n              lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n              // If a new replication slot was created in anticipation of forking,\n              // cancel it\n              destination?.initialSession.destroy();\n              return {\n                restoreFrom: replica,\n                replicateTo: reserved.replica,\n                // The slot is inactive, so its position is fixed.\n                slotLSN: replica.confirmedFlushLsn,\n                slotSession: reserved.reservation,\n              };\n            }\n          }\n        }\n      }\n    }\n  } catch (e) {\n    destination?.initialSession.destroy();\n    throw e;\n  }\n}\n\nasync function checkAndUpdateUpstream(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  {\n    replicaVersion,\n    publications: subscribed,\n    initialSyncContext,\n  }: SubscriptionStateAndContext,\n  replicaID: string | undefined,\n) {\n  const upstreamReplica = await getReplicaAtVersion(\n    lc,\n    sql,\n    shard,\n    replicaVersion,\n    replicaID,\n    initialSyncContext,\n  );\n  if (!upstreamReplica) {\n    throw new AutoResetSignal(\n      `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n    );\n  }\n\n  // Verify that the publications match what is being replicated.\n  const requested = shard.publications.toSorted();\n  const replicated = upstreamReplica.publications\n    .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n    .sort();\n  if (!deepEqual(requested, replicated)) {\n    lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n    await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n    throw new AutoResetSignal(\n      `Requested publications [${requested}] do not match configured ` +\n        `publications: [${replicated}]`,\n    );\n  }\n\n  // Sanity check: The subscription state on the replica should have the\n  // same publications. This should be guaranteed by the equivalence of the\n  // replicaVersion, but it doesn't hurt to verify.\n  if (!deepEqual(upstreamReplica.publications, subscribed)) {\n    throw new AutoResetSignal(\n      `Upstream publications [${upstreamReplica.publications}] do not ` +\n        `match subscribed publications [${subscribed}]`,\n    );\n  }\n\n  // Verify that the publications exist.\n  const exists = await sql`\n    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n  `.values();\n  if (exists.length !== subscribed.length) {\n    throw new AutoResetSignal(\n      `Upstream publications [${exists.flat()}] do not contain ` +\n        `all subscribed publications [${subscribed}]`,\n    );\n  }\n\n  const {slot} = upstreamReplica;\n  const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n  /*sql*/ `\n    SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n      WHERE slot_name = ${slot}`;\n  if (result.length === 0) {\n    throw new AutoResetSignal(`replication slot ${slot} is missing`);\n  }\n  const [{restartLSN, walStatus}] = result;\n  if (restartLSN === null || walStatus === 'lost') {\n    throw new AutoResetSignal(\n      `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n    );\n  }\n  const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n    SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n  return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAgEA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,iCACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;AAChC,GACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,MAAM,EACJ,SAAS,iBACT,WACA,uBACA,oBACE,iBACA,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,MAAM,eAAe,SAAS,QAAQ;GAMtC,MAAM,WAAW,QAAQ;GACzB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;GAAQ,GAChB,eAAe,OACjB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAIA,MAAM,mBAAmB,YAAY,CAAC;EACtC,MAAM,aAAa,uBAEf,qBAAqB,aAIrB,WACG,iBAAiB,cAAc,OAAO,KAAK,IAAI,CAAC,IAChD,iBAAiB,cAAc;EAqBtC,OAAO;GACL;GACA,cAAA,IArBuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;GAAQ,GACrB,SACA,qBACA,YAAY,UACZ,sBAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;GACjD;GACA;GACA,WAAW,iBAAiB;EAC9B;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAWA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,kBAAkB,WACL;CAM1B,MAAM,OAAO,MAAM,mBAAmB;CACtC,OAAO,SAAS,eAAe,8BAA8B;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBAAgB,gBAAgB,WAAW,kBAAkB,WACjE;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D,OAAO;GAAC,SAAS,KAAA;GAAW;EAAS;CACvC;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,cAAc,mBAAmB,WAAW,OAAO;EACzD,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EAWA,OAAO;GAAC;GAAS;GAAW,iBAAA,MAVE,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,GACJ,OACF;EAC2C;CAC7C;CACA,OAAO;EAAC;EAAS;CAAS;AAC5B;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,kBAAkB,WAC/B,eAC0B;CAC1B,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QAEH,OAAO;EAAC,SAAS,KAAA;EAAW,WAAW;CAAI;CAE7C,MAAM,EAAC,aAAa,aAAa,SAAS,gBAAe;CAKzD,SAAS,UAAU,oBAAoB,YAAY,cACjD,YAAY,QAAQ,CACtB;CAkBA,MAAM,OAAO,mBACT,MAAM,iBAAiB,qBAAqB,OAAO,CAAC,IACpD;CACJ,MAAM,wBAAwB,SAAS;CACvC,MAAM,YAAY,SAAS,gBAAgB,OAAO;CAIlD,MAAM,cAAc,mBAAmB,WAAW,WAAW;CAE7D,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EAWA,OAAO;GACL,SAAS;GACT;GACA;GACA,iBAAA,MAd4B,eAC5B,IACA;IAAC,GAAG;IAAY;GAAS,GACzB,aACA,aACA,WAAW,aACP;IAAC,MAAM;IAAQ,SAAS;GAAa,IACrC,KAAA,GACJ,OACF;EAMA;CACF;CACA,OAAO;EAAC,SAAS;EAAa;EAAW;CAAqB;AAChE;AAEA,SAAS,mBACP,WACA,EAAC,cACmB;CACpB,OAAO,aAAa;EAAC,gBAAgB;EAAY,cAAc;CAAE;AACnE;AAEA,eAAe,cAAc,KAA+B;CAC1D,MAAM,CAAC,EAAC,SAAQ,MAAM,GAAiB;CACvC,OAAO;AACT;AAEA,IAAM,2BAA2B;AACjC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BAcjB;CACA,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;CAEJ,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,IAAI,KAAK,OAAO,KAAK;GACjE,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,aAAa,eAAe,QAAQ;IACpC;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IAAI,IAAI,OAAO,GACb,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;MACZ,SACM,aACN,CACF;MAQA,YAAY,MAAM,cAAc,GAAG;MACnC,IAAI,SAAS,iBAAiB,IAAI,SAAS,OAAO,GAAG;OACnD,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,QAAQ,UAAU,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,iBAAiB,IAC7I,EAAC,QAAO,CACV;OACA;MACF;KACF;KAOA,IACE,eACA,WACA,SAAS,iBAAiB,KAAK,SAAS,OAAO,GAC/C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OACL,aAAa;OACb,aATkB,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAIlC;OACA,SAAS,YAAY,KAAK;OAC1B,aAAa,YAAY;MAC3B;KACF;IACF;IAEA,IACE,QAAQ,UAAU,KAClB,QAAQ,UACR,QAAQ,OAAO,aAAa,QAAQ;SAGhC,IAAI,OAAO,GACb,GAAG,OACD,uBAAuB,QAAQ,GAAG,GAAG,QAAQ,KAAK,kCAClD,EAAC,QAAO,CACV;IAAA;IAIJ,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,wBAAwB,WAAW,KACzE,EAAC,QAAO,CACV;KACA,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,OACF;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,aAAa,eAAe,QAAQ;OACpC,OAAO;QACL,aAAa;QACb,aAAa,SAAS;QAEtB,SAAS,QAAQ;QACjB,aAAa,SAAS;OACxB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,aAAa,eAAe,QAAQ;EACpC,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts

@@ -57,7 +57,7 @@ export declare class LagReporter {
     #private;
     static readonly MESSAGE_SUFFIX = "/lag-report/v1";
     readonly messagePrefix: string;
-    constructor(lc: LogContext, shard: ShardID, db: PostgresDB, pgVersion: number, lagIntervalMs: number);
+    constructor(lc: LogContext, shard: ShardID, db: PostgresDB, pgVersion: number, reportIntervalMs: number, reportIntervalJitterMs?: number);
     get pgVersion(): number;
     initiateLagReport(log?: boolean): Promise<{
         firstCommitTimeMs: number;

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAEV,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAA4B,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AAChF,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAKtE,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AACpD,OAAO,KAAK,EAAC,YAAY,EAAE,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EACV,eAAe,EACf,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAC,KAAK,aAAa,EAAC,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAC,SAAS,EAAqB,MAAM,iCAAiC,CAAC;AAK9E,OAAO,EAIL,KAAK,iCAAiC,EACvC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAKL,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,OAAO,EACb,MAAM,mBAAmB,CAAC;AAU3B;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,YAAY;;IAgBvD,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,OAAO,EAChB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,aAAa,EACtB,mBAAmB,EAAE,MAAM,EAC3B,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,EAC9B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,EAG3C,IAAI,GAAE;QACJ,SAAS,CAAC,EAAE,OAAO,SAAS,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,cAAc,CAAC;KACnC,EAwCP;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAI1B;IAEK,gBAAgB;;;cAyBrB;IAEK,WAAW,CACf,eAAe,EAAE,MAAM,EACvB,gBAAgB,GAAE,eAAe,EAAO,GACvC,OAAO,CAAC,YAAY,CAAC,CAqBvB;IAGK,mBAAmB,CACvB,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,WAAW,EAAE,mBAAmB,EAChC,gBAAgB,EAAE,eAAe,EAAE,GAClC,OAAO,CAAC,YAAY,CAAC,CAgKvB;CAiFF;AAGD,qBAAa,KAAM,YAAW,QAAQ;;IAIpC;;;;;;;OAOG;IACH,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI,EAG7D;IAED,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAyB1C;IAMD,GAAG,CAAC,SAAS,EAAE,WAAW,QAQzB;CAYF;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAoB/B,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAOtB;IAED,IAAI,SAAS,WAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ;;;OAuElC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS,CAYhE;IAED,IAAI,SAGH;IAwCD,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB,CAUpE;CAsCF;AA2rBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E;AAsBD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,kBAAkB,EACxB,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,EAC1D,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,GACzD,OAAO,CAwET"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAEV,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAA4B,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AAChF,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAKtE,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AACpD,OAAO,KAAK,EAAC,YAAY,EAAE,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EACV,eAAe,EACf,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAC,KAAK,aAAa,EAAC,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAC,SAAS,EAAqB,MAAM,iCAAiC,CAAC;AAK9E,OAAO,EAIL,KAAK,iCAAiC,EACvC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAKL,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,OAAO,EACb,MAAM,mBAAmB,CAAC;AAU3B;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,YAAY;;IAgBvD,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,OAAO,EAChB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,aAAa,EACtB,mBAAmB,EAAE,MAAM,EAC3B,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,EAC9B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,EAG3C,IAAI,GAAE;QACJ,SAAS,CAAC,EAAE,OAAO,SAAS,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,cAAc,CAAC;KACnC,EAwCP;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAI1B;IAEK,gBAAgB;;;cAyBrB;IAEK,WAAW,CACf,eAAe,EAAE,MAAM,EACvB,gBAAgB,GAAE,eAAe,EAAO,GACvC,OAAO,CAAC,YAAY,CAAC,CAqBvB;IAGK,mBAAmB,CACvB,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,WAAW,EAAE,mBAAmB,EAChC,gBAAgB,EAAE,eAAe,EAAE,GAClC,OAAO,CAAC,YAAY,CAAC,CAgKvB;CAiFF;AAGD,qBAAa,KAAM,YAAW,QAAQ;;IAIpC;;;;;;;OAOG;IACH,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI,EAG7D;IAED,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAyB1C;IAMD,GAAG,CAAC,SAAS,EAAE,WAAW,QAQzB;CAYF;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAwB/B,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,MAAM,EACjB,gBAAgB,EAAE,MAAM,EACxB,sBAAsB,GAAE,MAAa,EAQtC;IAED,IAAI,SAAS,WAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ;;;OAuElC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS,CAYhE;IAED,IAAI,SAKH;IA6CD,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB,CAUpE;CAwDF;AA2rBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E;AAsBD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,kBAAkB,EACxB,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,EAC1D,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,GACzD,OAAO,CAwET"}

package/out/zero-cache/src/services/change-source/pg/change-source.js

@@ -290,20 +290,23 @@ var LagReporter = class LagReporter {
 	#lc;
 	messagePrefix;
 	#db;
-	#lagIntervalMs;
+	#reportIntervalMs;
+	#reportIntervalJitterMs;
 	#lagReportRetries = getOrCreateCounter("replication", "lag_report_retries", {
 		description: "Number of replication lag reports retried because the expected report was not received before the next report interval.",
 		unit: "{report}"
 	});
 	#pgVersion;
 	#expectingLagReport = null;
+	#lastAcceptedSendTimeMs = 0;
 	#timer;
-	constructor(lc, shard, db, pgVersion, lagIntervalMs) {
+	constructor(lc, shard, db, pgVersion, reportIntervalMs, reportIntervalJitterMs = 2e3) {
 		this.#lc = lc;
 		this.messagePrefix = `${shard.appID}/${shard.shardNum}${LagReporter.MESSAGE_SUFFIX}`;
 		this.#db = db;
 		this.#pgVersion = pgVersion;
-		this.#lagIntervalMs = lagIntervalMs;
+		this.#reportIntervalMs = reportIntervalMs;
+		this.#reportIntervalJitterMs = reportIntervalJitterMs;
 	}
 	get pgVersion() {
 		return this.#pgVersion;
@@ -392,8 +395,12 @@ var LagReporter = class LagReporter {
 	stop() {
 		clearTimeout(this.#timer);
 		this.#timer = void 0;
+		this.#expectingLagReport = null;
+		this.#lastAcceptedSendTimeMs = 0;
 	}
-	#scheduleNextReport(delayMs) {
+	#scheduleNextReportWithJitter(delayMs) {
+		const jitter = Math.floor(Math.random() * this.#reportIntervalJitterMs) - this.#reportIntervalJitterMs / 2;
+		delayMs = Math.max(0, delayMs + jitter);
 		this.#expectingLagReport = null;
 		clearTimeout(this.#timer);
 		this.#timer = setTimeout(async () => {
@@ -402,7 +409,7 @@ var LagReporter = class LagReporter {
 				await this.initiateLagReport();
 			} catch (e) {
 				this.#lc.warn?.(`error initiating lag report`, e);
-				this.#scheduleNextReport(this.#lagIntervalMs);
+				this.#scheduleNextReportWithJitter(this.#reportIntervalMs);
 			}
 		}, delayMs);
 	}
@@ -422,9 +429,9 @@ var LagReporter = class LagReporter {
 				await this.initiateLagReport();
 			} catch (e) {
 				this.#lc.warn?.(`error retrying lag report`, e);
-				this.#scheduleNextReport(this.#lagIntervalMs);
+				this.#scheduleNextReportWithJitter(this.#reportIntervalMs);
 			}
-		}, this.#lagIntervalMs);
+		}, this.#reportIntervalMs);
 	}
 	processLagReportMessage(msg) {
 		assert(msg.prefix === this.messagePrefix, `unexpected message prefix: ${msg.prefix}`);
@@ -435,10 +442,11 @@ var LagReporter = class LagReporter {
 		const now = Date.now();
 		const expectedReport = this.#expectingLagReport;
 		let nextSendTimeMs;
-		if (report.id === expectedReport?.id) {
-			nextSendTimeMs = Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
-			this.#scheduleNextReport(nextSendTimeMs - now);
-		} else nextSendTimeMs = expectedReport?.sendTimeMs ?? Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
+		if (expectedReport ? Math.abs(report.sendTimeMs - expectedReport.sendTimeMs) < this.#reportIntervalMs : this.#timer !== void 0 && report.sendTimeMs > this.#lastAcceptedSendTimeMs) {
+			this.#lastAcceptedSendTimeMs = report.sendTimeMs;
+			nextSendTimeMs = Math.max(now, report.sendTimeMs + this.#reportIntervalMs);
+			this.#scheduleNextReportWithJitter(nextSendTimeMs - now);
+		} else nextSendTimeMs = expectedReport?.sendTimeMs ?? Math.max(now, report.sendTimeMs + this.#reportIntervalMs);
 		const { sendTimeMs, commitTimeMs } = report;
 		return [
 			"status",