@rocicorp/zero 1.12.0-canary.4 → 1.12.0-canary.6

package/out/zero-cache/src/server/change-streamer.js

@@ -5,9 +5,9 @@ import { broadcastWorker, childWorker, parentWorker, singleProcessMode } from ".
 import { getShardConfig } from "../types/shards.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { DatabaseInitError } from "../../../zqlite/src/db.js";
-import { getOrCreateGauge } from "../observability/metrics.js";
 import { connectPgClient } from "../types/pg.js";
 import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruption.js";
+import { getOrCreateGauge } from "../observability/metrics.js";
 import { SupportingProcesses, exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { REPLICATOR_URL } from "./worker-urls.js";
@@ -31,6 +31,7 @@ import { sqliteFileBytes } from "../services/replicator/sqlite-change-log-observ
 import { PurgeLocker } from "../services/change-streamer/storer.js";
 import { initializeStreamer } from "../services/change-streamer/change-streamer-service.js";
 import { initChangeStreamerSchema } from "../services/change-streamer/schema/init.js";
+import { ServerStartMetrics } from "../services/change-streamer/server-start-metrics.js";
 import { startOtelAuto } from "./otel-start.js";
 import { resolver } from "@rocicorp/resolver";
 import { LogContext, consoleLogSink } from "@rocicorp/logger";
@@ -56,6 +57,7 @@ async function runWorker(parent, env, ...argv) {
 	const changeDB = pgChangeLogEnabled ? await connectChangeDB(lc, change.db, shard) : null;
 	let purgeLock = null;
 	let changeStreamer;
+	let backupMonitor;
 	let backupURL;
 	const context = getServerContext(config);
 	const sqliteChangeLogEnabled = sqliteChangeLogMode !== "off";
@@ -71,6 +73,7 @@ async function runWorker(parent, env, ...argv) {
 			unit: "By"
 		}).addCallback(async (o) => o.observe(await sqliteFileBytes(lc, changeLogFile)));
 	}
+	const startMetrics = new ServerStartMetrics();
 	let waitForFirstBackupBeforeServing = false;
 	let newBackupLineage = false;
 	for (const first of [true, false]) {
@@ -93,7 +96,7 @@ async function runWorker(parent, env, ...argv) {
 		};
 		try {
 			const { pgReplicationEpoch: epoch, pgReplicationSlotPerReplica: slotPerReplica, pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs } = upstream;
-			const { changeSource, subscriptionState, destinationBackupURL, replicaID, waitForBackupBeforeServing, newBackupLineage: initNewBackupLineage, pgChangeLogBehindSlot } = upstream.type === "pg" ? await initializePostgresChangeSource(lc, upstream.db, shard, replica.file, {
+			const { changeSource, subscriptionState, destinationBackupURL, replicaID, waitForBackupBeforeServing, newBackupLineage: initNewBackupLineage, pgChangeLogBehindSlot, startTimings } = upstream.type === "pg" ? await initializePostgresChangeSource(lc, upstream.db, shard, replica.file, {
 				...initialSync,
 				replicationSlotFailover: upstream.pgReplicationSlotFailover,
 				installPartialIndexTriggers: upstream.pgPartialIndexTriggers
@@ -105,6 +108,17 @@ async function runWorker(parent, env, ...argv) {
 				pgChangeLog: pgChangeLogEnabled
 			}, upstream.pgStreamInboundTimeoutMs) : await initializeCustomChangeSource(lc, upstream.db, shard, replica.file, context, restoreOptions);
 			const replicationStatusPublisher = ReplicationStatusPublisher.forReplicaFile(replica.file);
+			backupMonitor = createBackupCleanupMonitor({
+				lc,
+				config: destinationBackupURL ? {
+					...config,
+					litestream: {
+						...config.litestream,
+						backupURLOverride: destinationBackupURL
+					}
+				} : config,
+				replicaFile: replica.file
+			});
 			changeStreamer = await initializeStreamer(lc, shard, taskID, address, protocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, destinationBackupURL ? {
 				backupURL: destinationBackupURL,
 				litestreamVersion: litestream.backupUsingV5 ? "v5" : "legacy",
@@ -145,14 +159,20 @@ async function runWorker(parent, env, ...argv) {
 					readPercent: sqliteChangeLogReadPercent,
 					coldReadPercent: sqliteChangeLogColdReadPercent,
 					retentionMs: sqliteChangeLogRetentionMs
-				} : void 0
+				} : void 0,
+				backupMonitor
 			}, setTimeout);
 			backupURL = destinationBackupURL;
 			waitForFirstBackupBeforeServing = waitForBackupBeforeServing;
+			startMetrics.setKind(startTimings?.kind);
+			startMetrics.record("fork_wait", startTimings?.forkWaitMs);
+			startMetrics.record("restore", startTimings?.restoreMs);
 			newBackupLineage = initNewBackupLineage;
 			break;
 		} catch (e) {
 			await cleanup.release();
+			backupMonitor?.stop();
+			backupMonitor = void 0;
 			if (first && e instanceof AutoResetSignal) {
 				lc.warn?.(`resetting replica ${replica.file}`, e);
 				deleteLiteDB(replica.file);
@@ -164,9 +184,14 @@ async function runWorker(parent, env, ...argv) {
 		}
 	}
 	assert(changeStreamer, `resetting replica did not advance replicaVersion`);
+	assert(backupMonitor, `backup monitor is created with the change-streamer`);
 	const processes = new SupportingProcesses(lc, parent);
 	const profileSubWorkers = [];
 	const { promise: replicatorReady, resolve } = resolver();
+	if (backupURL) {
+		const catchUpStart = performance.now();
+		replicatorReady.then(() => startMetrics.record("catch_up", performance.now() - catchUpStart));
+	}
 	if (!backupURL) resolve();
 	else {
 		lc.info?.("setting up backup to", backupURL);
@@ -179,12 +204,6 @@ async function runWorker(parent, env, ...argv) {
 		profileSubWorkers.push(backupReplicator);
 		backupReplicator.onMessageType("profileResponse", (res) => parent.send(["profileResponse", res]));
 	}
-	const backupMonitor = createBackupCleanupMonitor({
-		lc,
-		config,
-		replicaFile: replica.file,
-		changeStreamer
-	});
 	let backupReady = promiseVoid;
 	if (waitForFirstBackupBeforeServing) {
 		const start = performance.now();
@@ -192,6 +211,7 @@ async function runWorker(parent, env, ...argv) {
 		backupReady = backupMonitor.firstBackupReceived().then(() => {
 			const elapsed = performance.now() - start;
 			lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);
+			startMetrics.record("backup_confirm", elapsed);
 		});
 	}
 	const readinessGate = pgChangeLogEnabled ? backupReady : Promise.all([backupReady, replicatorReady]);
@@ -204,7 +224,10 @@ async function runWorker(parent, env, ...argv) {
 		config,
 		getProfileWorker: profileWorker ? () => Promise.resolve(profileWorker) : void 0
 	}, parent, changeStreamer);
-	readinessGate.then(() => parent.send(["ready", { ready: true }]));
+	readinessGate.then(() => {
+		startMetrics.ready();
+		parent.send(["ready", { ready: true }]);
+	});
 	running = true;
 	await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor, processes);
 }

package/out/zero-cache/src/server/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {InitCleanup} from '../services/change-source/common/init-cleanup.ts';\nimport type {PgChangeLogPurgeLocker} from '../services/change-source/common/replica-restore.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {DEFAULT_READINESS_GRACE_PERIOD_MS} from '../services/change-streamer/restore-admission.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {\n  PurgeLocker,\n  type PurgeLock,\n} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {deleteLitestreamMetaDir} from '../services/litestream/commands.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {warnIfPostgresUnpatched} from '../types/pg-patch.ts';\nimport {connectPgClient, type PostgresDB} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig, type ShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\n// Errors thrown once the services are running are reported by the services\n// themselves (e.g. as 'Replicating' status events), and are thus not\n// published as 'Initializing' errors.\nlet running = false;\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      maxConcurrentSwitchoverRestores,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n  if (upstream.type === 'pg') {\n    warnIfPostgresUnpatched(lc);\n  }\n\n  const shard = getShardConfig(config);\n\n  // The change DB is only used by the PG change log. When it is disabled\n  // (RMv2), no connection to the change DB is made.\n  const changeDB = pgChangeLogEnabled\n    ? await connectChangeDB(lc, change.db, shard)\n    : null;\n\n  let purgeLock = null as PurgeLock | null;\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  let waitForFirstBackupBeforeServing = false;\n  let newBackupLineage = false;\n  for (const first of [true, false]) {\n    // Resources acquired by an initialization attempt (e.g. a claimed\n    // replication slot, the purge lock) to be released if the attempt fails.\n    const cleanup = new InitCleanup(lc);\n    // When restoring from litestream, a lock is acquired to prevent change-log\n    // purges. This ensures that (this) change-streamer will be able to resume\n    // from the backup. The lock is acquired by the change source initialization\n    // just before restoring (i.e. after a replication slot is created, if\n    // applicable), and released once this change-streamer takes over the\n    // change-log.\n    const acquirePurgeLock: PgChangeLogPurgeLocker | undefined =\n      changeDB && litestream.backupURL && litestream.executable\n        ? async (slotWatermark?: string) => {\n            const lock = await new PurgeLocker(lc, shard, changeDB).acquire(\n              slotWatermark,\n            );\n            purgeLock = lock === 'behind-slot' ? null : lock;\n            if (purgeLock) {\n              const acquired = purgeLock;\n              cleanup.onFailure('purge lock', () => {\n                purgeLock = null;\n                return acquired.release();\n              });\n            }\n            return lock;\n          }\n        : undefined;\n\n    const restoreOptions = {litestream, acquirePurgeLock, cleanup};\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        pgReplicationEpoch: epoch,\n        pgReplicationSlotPerReplica: slotPerReplica,\n        pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n      } = upstream;\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n        newBackupLineage: initNewBackupLineage,\n        pgChangeLogBehindSlot,\n      } = upstream.type === 'pg'\n        ? await initializePostgresChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            {\n              ...initialSync,\n              replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n            },\n            context,\n            replicationLag.reportIntervalMs,\n            restoreOptions,\n            {\n              epoch,\n              slotPerReplica,\n              inactiveReplicaGracePeriodMs,\n              backupV5: litestream.backupUsingV5,\n              pgChangeLog: pgChangeLogEnabled,\n            },\n            upstream.pgStreamInboundTimeoutMs,\n          )\n        : await initializeCustomChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            context,\n            restoreOptions,\n          );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n              replicaFile: replica.file,\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogBehindSlot: pgChangeLogBehindSlot ?? false,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          maxConcurrentSwitchoverRestores,\n          // When lag reporting is enabled, a restored view-syncer waits for\n          // the next lag report (sent at an interval with ~1 second of\n          // jitter) before becoming ready.\n          switchoverRestoreReadinessGracePeriodMs:\n            (replicationLag.reportIntervalMs > 0\n              ? replicationLag.reportIntervalMs + 1000\n              : 0) + DEFAULT_READINESS_GRACE_PERIOD_MS,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      newBackupLineage = initNewBackupLineage;\n      break;\n    } catch (e) {\n      // Release the resources acquired by the failed attempt, e.g.:\n      // * The claimed replication slot, which would otherwise remain active\n      //   indefinitely, preventing a retry from resuming (or forking) it.\n      // * The purge lock. This is safe because the purge lock exists to\n      //   preserve change-log entries so the new change-streamer can resume\n      //   from the backup replica's watermark. An AutoResetSignal means we\n      //   can't resume from the backup replica (e.g. its replication slot is\n      //   gone), so the change-log entries the lock was protecting are no\n      //   longer needed. The retry performs a fresh initial sync with a new\n      //   replication slot, independent of the old change-log. Releasing is\n      //   also necessary to avoid a self-deadlock when CHANGE_DB ==\n      //   UPSTREAM_DB: CREATE_REPLICATION_SLOT waits for all older\n      //   transactions to finish, including this lock's open transaction.\n      await cleanup.release();\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  // The backup-replicator is stopped with the other services, i.e. after the\n  // ChangeStreamerHttpServer has drained its subscribers.\n  const processes = new SupportingProcesses(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  const {promise: replicatorReady, resolve} = resolver();\n  if (!backupURL) {\n    resolve(); // No backup replicator to wait for, so resolve immediately.\n  } else {\n    lc.info?.('setting up backup to', backupURL);\n    config.litestream.backupURLOverride = backupURL;\n    if (newBackupLineage) {\n      deleteLitestreamMetaDir(replica.file);\n    }\n    // Start a backup replicator, which starts up the corresponding\n    // litestream backup process.\n    const backupReplicator = processes\n      .add(\n        childWorker(\n          REPLICATOR_URL,\n          {...env, ZERO_LITESTREAM_BACKUP_URL_OVERRIDE: backupURL},\n          'backup' satisfies ReplicaFileMode,\n        ),\n        'backup-replicator',\n      )\n      .onceMessageType('ready', () => resolve());\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n  }\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n  });\n\n  let backupReady = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    backupReady = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n    });\n  }\n\n  // In RMv2, readiness additionally waits for the backup-replicator to catch\n  // up to the replication stream. This is not done in RMv1 (i.e. when the\n  // PG change-log is enabled), in which the change-streamer itself is only\n  // started after the readinessGate (plus a startup delay), and thus the\n  // backup-replicator cannot catch up before then.\n  const readinessGate = pgChangeLogEnabled\n    ? backupReady\n    : Promise.all([backupReady, replicatorReady]);\n\n  // Create the broadcast facade once: each broadcastWorker() adds permanent\n  // 'message' forwarders to every sub worker, so creating one per /profz\n  // request would leak a forwarder per request.\n  const profileWorker =\n    profileSubWorkers.length > 0\n      ? broadcastWorker(profileSubWorkers)\n      : undefined;\n  const getProfileWorker = profileWorker\n    ? () => Promise.resolve(profileWorker)\n    : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      // The startup delay is only relevant when taking over the PG change-log\n      // (RMv1 and RMv1.5), and is disabled for RMv2.\n      startupDelayMs: pgChangeLogEnabled ? startupDelayMs : 0,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  running = true;\n  await runUntilKilled(\n    lc,\n    parent,\n    changeStreamerWebServer,\n    backupMonitor,\n    processes,\n  );\n}\n\n/**\n * Connects the startup-time client used for change-streamer initialization\n * and handoff / takeover, and ensures that the change DB schema is\n * initialized / up-to-date. Steady-state clients are managed by the\n * change-streamer (Storer) implementation.\n */\nasync function connectChangeDB(\n  lc: LogContext,\n  connectionURI: string,\n  shard: ShardConfig,\n): Promise<PostgresDB> {\n  const changeDB = await connectPgClient(\n    lc,\n    connectionURI,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  return changeDB;\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        if (!running) {\n          await publishCriticalEvent(\n            lc,\n            replicationStatusError(lc, 'Initializing', e),\n          );\n        }\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAKlD,IAAI,UAAU;AAEd,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,iCACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CACxB,IAAI,SAAS,SAAS,MACpB,wBAAwB,EAAE;CAG5B,MAAM,QAAQ,eAAe,MAAM;CAInC,MAAM,WAAW,qBACb,MAAM,gBAAgB,IAAI,OAAO,IAAI,KAAK,IAC1C;CAEJ,IAAI,YAAY;CAChB,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,IAAI,mBAAmB;CACvB,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAAG;EAGjC,MAAM,UAAU,IAAI,YAAY,EAAE;EAyBlC,MAAM,iBAAiB;GAAC;GAAY,kBAjBlC,YAAY,WAAW,aAAa,WAAW,aAC3C,OAAO,kBAA2B;IAChC,MAAM,OAAO,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,CAAC,CAAC,QACtD,aACF;IACA,YAAY,SAAS,gBAAgB,OAAO;IAC5C,IAAI,WAAW;KACb,MAAM,WAAW;KACjB,QAAQ,UAAU,oBAAoB;MACpC,YAAY;MACZ,OAAO,SAAS,QAAQ;KAC1B,CAAC;IACH;IACA,OAAO;GACT,IACA,KAAA;GAEgD;EAAO;EAC7D,IAAI;GAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;GACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,4BACA,kBAAkB,sBAClB,0BACE,SAAS,SAAS,OAClB,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;IACE,GAAG;IACH,yBAAyB,SAAS;IAClC,6BAA6B,SAAS;GACxC,GACA,SACA,eAAe,kBACf,gBACA;IACE;IACA;IACA;IACA,UAAU,WAAW;IACrB,aAAa;GACf,GACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;GAEJ,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;GAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;IACE,WAAW;IACX,mBAAmB,WAAW,gBAAgB,OAAO;IACrD,aAAa,QAAQ;GACvB,IACA,MACJ,WACA,aAAa,OACb;IACE,uBAAuB,yBAAyB;IAChD;IACA;IACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;IACN;IAIA,0CACG,eAAe,mBAAmB,IAC/B,eAAe,mBAAmB,MAClC,KAAK;IACX,oBAAoB,OAAO;IAC3B,oBAAoB,OAAO;IAC3B,eAAe;KACb,eAAe,kBAAkB,QAAQ,IAAI;KAC7C,eAAe;KACf,kBAAkB;IACpB;IAIA,uBAAuB,yBACnB;KACE,aAAa,QAAQ;KACrB,UAAU;MAGR,OAAO;MACP,YAAY,kBAAkB;MAC9B;KACF;IACF,IACA,KAAA;IAIJ,sBAAsB,yBAClB;KACE,aAAa;KACb,WAAW;IACb,IACA,KAAA;IAEJ,wBACE,sBAAsB,2BAClB;KACE,aAAa,QAAQ;KACrB,gBAAgB;KAChB,aAAa;KACb,eAAe;IACjB,IACA,KAAA;IAIN,sBACE,wBAAwB,UACpB;KACE,aAAa;KACb,iBAAiB;KACjB,aAAa;IACf,IACA,KAAA;GACR,GACA,UACF;GACA,YAAY;GACZ,kCAAkC;GAClC,mBAAmB;GACnB;EACF,SAAS,GAAG;GAcV,MAAM,QAAQ,QAAQ;GACtB,IAAI,SAAS,aAAa,iBAAiB;IACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;IAGhD,aAAa,QAAQ,IAAI;IAMzB,kBAAkB,QAAQ,IAAI;IAC9B;GACF;GACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;GAEF,MAAM;EACR;CACF;CAEA,OAAO,gBAAgB,kDAAkD;CAIzE,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,MAAM,oBAA8B,CAAC;CACrC,MAAM,EAAC,SAAS,iBAAiB,YAAW,SAAS;CACrD,IAAI,CAAC,WACH,QAAQ;MACH;EACL,GAAG,OAAO,wBAAwB,SAAS;EAC3C,OAAO,WAAW,oBAAoB;EACtC,IAAI,kBACF,wBAAwB,QAAQ,IAAI;EAItC,MAAM,mBAAmB,UACtB,IACC,YACE,gBACA;GAAC,GAAG;GAAK,qCAAqC;EAAS,GACvD,QACF,GACA,mBACF,CAAC,CACA,gBAAgB,eAAe,QAAQ,CAAC;EAC3C,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;CACF;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,cAAc;CAClB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAc,cAAc,oBAAoB,CAAC,CAAC,WAAW;GAC3D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAOA,MAAM,gBAAgB,qBAClB,cACA,QAAQ,IAAI,CAAC,aAAa,eAAe,CAAC;CAK9C,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAGA,gBAAgB,qBAAqB,iBAAiB;EACtD;EACA;EACA,kBAdqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAaF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,UAAU;CACV,MAAM,eACJ,IACA,QACA,yBACA,eACA,SACF;AACF;;;;;;;AAQA,eAAe,gBACb,IACA,eACA,OACqB;CACrB,MAAM,WAAW,MAAM,gBACrB,IACA,eACA,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC;CAC7D,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,OAAO;AACT;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,CAAC,CAAC,MAAM,OAAM,MAAK;CACjB,IAAI,CAAC,SACH,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CAEF,MAAM;AACR,CAAC,CACL"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {InitCleanup} from '../services/change-source/common/init-cleanup.ts';\nimport type {PgChangeLogPurgeLocker} from '../services/change-source/common/replica-restore.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport type {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {DEFAULT_READINESS_GRACE_PERIOD_MS} from '../services/change-streamer/restore-admission.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {ServerStartMetrics} from '../services/change-streamer/server-start-metrics.ts';\nimport {\n  PurgeLocker,\n  type PurgeLock,\n} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {deleteLitestreamMetaDir} from '../services/litestream/commands.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {warnIfPostgresUnpatched} from '../types/pg-patch.ts';\nimport {connectPgClient, type PostgresDB} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig, type ShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\n// Errors thrown once the services are running are reported by the services\n// themselves (e.g. as 'Replicating' status events), and are thus not\n// published as 'Initializing' errors.\nlet running = false;\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      maxConcurrentSwitchoverRestores,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n  if (upstream.type === 'pg') {\n    warnIfPostgresUnpatched(lc);\n  }\n\n  const shard = getShardConfig(config);\n\n  // The change DB is only used by the PG change log. When it is disabled\n  // (RMv2), no connection to the change DB is made.\n  const changeDB = pgChangeLogEnabled\n    ? await connectChangeDB(lc, change.db, shard)\n    : null;\n\n  let purgeLock = null as PurgeLock | null;\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupMonitor: BackupMonitor | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  const startMetrics = new ServerStartMetrics();\n  let waitForFirstBackupBeforeServing = false;\n  let newBackupLineage = false;\n  for (const first of [true, false]) {\n    // Resources acquired by an initialization attempt (e.g. a claimed\n    // replication slot, the purge lock) to be released if the attempt fails.\n    const cleanup = new InitCleanup(lc);\n    // When restoring from litestream, a lock is acquired to prevent change-log\n    // purges. This ensures that (this) change-streamer will be able to resume\n    // from the backup. The lock is acquired by the change source initialization\n    // just before restoring (i.e. after a replication slot is created, if\n    // applicable), and released once this change-streamer takes over the\n    // change-log.\n    const acquirePurgeLock: PgChangeLogPurgeLocker | undefined =\n      changeDB && litestream.backupURL && litestream.executable\n        ? async (slotWatermark?: string) => {\n            const lock = await new PurgeLocker(lc, shard, changeDB).acquire(\n              slotWatermark,\n            );\n            purgeLock = lock === 'behind-slot' ? null : lock;\n            if (purgeLock) {\n              const acquired = purgeLock;\n              cleanup.onFailure('purge lock', () => {\n                purgeLock = null;\n                return acquired.release();\n              });\n            }\n            return lock;\n          }\n        : undefined;\n\n    const restoreOptions = {litestream, acquirePurgeLock, cleanup};\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        pgReplicationEpoch: epoch,\n        pgReplicationSlotPerReplica: slotPerReplica,\n        pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n      } = upstream;\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n        newBackupLineage: initNewBackupLineage,\n        pgChangeLogBehindSlot,\n        startTimings,\n      } = upstream.type === 'pg'\n        ? await initializePostgresChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            {\n              ...initialSync,\n              replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n            },\n            context,\n            replicationLag.reportIntervalMs,\n            restoreOptions,\n            {\n              epoch,\n              slotPerReplica,\n              inactiveReplicaGracePeriodMs,\n              backupV5: litestream.backupUsingV5,\n              pgChangeLog: pgChangeLogEnabled,\n            },\n            upstream.pgStreamInboundTimeoutMs,\n          )\n        : await initializeCustomChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            context,\n            restoreOptions,\n          );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      // Created before the change-streamer, which tracks its watermarks (and\n      // requests prompt backups from it) from construction. It is run (and\n      // stopped) as a service below. It monitors the backup-replicator's\n      // destination (see `backupURLOverride` below).\n      backupMonitor = createBackupCleanupMonitor({\n        lc,\n        config: destinationBackupURL\n          ? {\n              ...config,\n              litestream: {\n                ...config.litestream,\n                backupURLOverride: destinationBackupURL,\n              },\n            }\n          : config,\n        replicaFile: replica.file,\n      });\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n              replicaFile: replica.file,\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogBehindSlot: pgChangeLogBehindSlot ?? false,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          maxConcurrentSwitchoverRestores,\n          // When lag reporting is enabled, a restored view-syncer waits for\n          // the next lag report (sent at an interval with ~1 second of\n          // jitter) before becoming ready.\n          switchoverRestoreReadinessGracePeriodMs:\n            (replicationLag.reportIntervalMs > 0\n              ? replicationLag.reportIntervalMs + 1000\n              : 0) + DEFAULT_READINESS_GRACE_PERIOD_MS,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n          backupMonitor,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      startMetrics.setKind(startTimings?.kind);\n      startMetrics.record('fork_wait', startTimings?.forkWaitMs);\n      startMetrics.record('restore', startTimings?.restoreMs);\n      newBackupLineage = initNewBackupLineage;\n      break;\n    } catch (e) {\n      // Release the resources acquired by the failed attempt, e.g.:\n      // * The claimed replication slot, which would otherwise remain active\n      //   indefinitely, preventing a retry from resuming (or forking) it.\n      // * The purge lock. This is safe because the purge lock exists to\n      //   preserve change-log entries so the new change-streamer can resume\n      //   from the backup replica's watermark. An AutoResetSignal means we\n      //   can't resume from the backup replica (e.g. its replication slot is\n      //   gone), so the change-log entries the lock was protecting are no\n      //   longer needed. The retry performs a fresh initial sync with a new\n      //   replication slot, independent of the old change-log. Releasing is\n      //   also necessary to avoid a self-deadlock when CHANGE_DB ==\n      //   UPSTREAM_DB: CREATE_REPLICATION_SLOT waits for all older\n      //   transactions to finish, including this lock's open transaction.\n      await cleanup.release();\n      // Not yet started (the change-streamer starts it), so nothing to wait on.\n      void backupMonitor?.stop();\n      backupMonitor = undefined;\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n  assert(backupMonitor, `backup monitor is created with the change-streamer`);\n\n  // The backup-replicator is stopped with the other services, i.e. after the\n  // ChangeStreamerHttpServer has drained its subscribers.\n  const processes = new SupportingProcesses(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  const {promise: replicatorReady, resolve} = resolver();\n  if (backupURL) {\n    const catchUpStart = performance.now();\n    void replicatorReady.then(() =>\n      startMetrics.record('catch_up', performance.now() - catchUpStart),\n    );\n  }\n  if (!backupURL) {\n    resolve(); // No backup replicator to wait for, so resolve immediately.\n  } else {\n    lc.info?.('setting up backup to', backupURL);\n    config.litestream.backupURLOverride = backupURL;\n    if (newBackupLineage) {\n      deleteLitestreamMetaDir(replica.file);\n    }\n    // Start a backup replicator, which starts up the corresponding\n    // litestream backup process.\n    const backupReplicator = processes\n      .add(\n        childWorker(\n          REPLICATOR_URL,\n          {...env, ZERO_LITESTREAM_BACKUP_URL_OVERRIDE: backupURL},\n          'backup' satisfies ReplicaFileMode,\n        ),\n        'backup-replicator',\n      )\n      .onceMessageType('ready', () => resolve());\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n  }\n\n  let backupReady = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    backupReady = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n      startMetrics.record('backup_confirm', elapsed);\n    });\n  }\n\n  // In RMv2, readiness additionally waits for the backup-replicator to catch\n  // up to the replication stream. This is not done in RMv1 (i.e. when the\n  // PG change-log is enabled), in which the change-streamer itself is only\n  // started after the readinessGate (plus a startup delay), and thus the\n  // backup-replicator cannot catch up before then.\n  const readinessGate = pgChangeLogEnabled\n    ? backupReady\n    : Promise.all([backupReady, replicatorReady]);\n\n  // Create the broadcast facade once: each broadcastWorker() adds permanent\n  // 'message' forwarders to every sub worker, so creating one per /profz\n  // request would leak a forwarder per request.\n  const profileWorker =\n    profileSubWorkers.length > 0\n      ? broadcastWorker(profileSubWorkers)\n      : undefined;\n  const getProfileWorker = profileWorker\n    ? () => Promise.resolve(profileWorker)\n    : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      // The startup delay is only relevant when taking over the PG change-log\n      // (RMv1 and RMv1.5), and is disabled for RMv2.\n      startupDelayMs: pgChangeLogEnabled ? startupDelayMs : 0,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => {\n    startMetrics.ready();\n    parent.send(['ready', {ready: true}]);\n  });\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  running = true;\n  await runUntilKilled(\n    lc,\n    parent,\n    changeStreamerWebServer,\n    backupMonitor,\n    processes,\n  );\n}\n\n/**\n * Connects the startup-time client used for change-streamer initialization\n * and handoff / takeover, and ensures that the change DB schema is\n * initialized / up-to-date. Steady-state clients are managed by the\n * change-streamer (Storer) implementation.\n */\nasync function connectChangeDB(\n  lc: LogContext,\n  connectionURI: string,\n  shard: ShardConfig,\n): Promise<PostgresDB> {\n  const changeDB = await connectPgClient(\n    lc,\n    connectionURI,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  return changeDB;\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        if (!running) {\n          await publishCriticalEvent(\n            lc,\n            replicationStatusError(lc, 'Initializing', e),\n          );\n        }\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAKlD,IAAI,UAAU;AAEd,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,iCACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CACxB,IAAI,SAAS,SAAS,MACpB,wBAAwB,EAAE;CAG5B,MAAM,QAAQ,eAAe,MAAM;CAInC,MAAM,WAAW,qBACb,MAAM,gBAAgB,IAAI,OAAO,IAAI,KAAK,IAC1C;CAEJ,IAAI,YAAY;CAChB,IAAI;CACJ,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,MAAM,eAAe,IAAI,mBAAmB;CAC5C,IAAI,kCAAkC;CACtC,IAAI,mBAAmB;CACvB,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAAG;EAGjC,MAAM,UAAU,IAAI,YAAY,EAAE;EAyBlC,MAAM,iBAAiB;GAAC;GAAY,kBAjBlC,YAAY,WAAW,aAAa,WAAW,aAC3C,OAAO,kBAA2B;IAChC,MAAM,OAAO,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,CAAC,CAAC,QACtD,aACF;IACA,YAAY,SAAS,gBAAgB,OAAO;IAC5C,IAAI,WAAW;KACb,MAAM,WAAW;KACjB,QAAQ,UAAU,oBAAoB;MACpC,YAAY;MACZ,OAAO,SAAS,QAAQ;KAC1B,CAAC;IACH;IACA,OAAO;GACT,IACA,KAAA;GAEgD;EAAO;EAC7D,IAAI;GAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;GACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,4BACA,kBAAkB,sBAClB,uBACA,iBACE,SAAS,SAAS,OAClB,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;IACE,GAAG;IACH,yBAAyB,SAAS;IAClC,6BAA6B,SAAS;GACxC,GACA,SACA,eAAe,kBACf,gBACA;IACE;IACA;IACA;IACA,UAAU,WAAW;IACrB,aAAa;GACf,GACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;GAEJ,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;GAMxD,gBAAgB,2BAA2B;IACzC;IACA,QAAQ,uBACJ;KACE,GAAG;KACH,YAAY;MACV,GAAG,OAAO;MACV,mBAAmB;KACrB;IACF,IACA;IACJ,aAAa,QAAQ;GACvB,CAAC;GAED,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;IACE,WAAW;IACX,mBAAmB,WAAW,gBAAgB,OAAO;IACrD,aAAa,QAAQ;GACvB,IACA,MACJ,WACA,aAAa,OACb;IACE,uBAAuB,yBAAyB;IAChD;IACA;IACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;IACN;IAIA,0CACG,eAAe,mBAAmB,IAC/B,eAAe,mBAAmB,MAClC,KAAK;IACX,oBAAoB,OAAO;IAC3B,oBAAoB,OAAO;IAC3B,eAAe;KACb,eAAe,kBAAkB,QAAQ,IAAI;KAC7C,eAAe;KACf,kBAAkB;IACpB;IAIA,uBAAuB,yBACnB;KACE,aAAa,QAAQ;KACrB,UAAU;MAGR,OAAO;MACP,YAAY,kBAAkB;MAC9B;KACF;IACF,IACA,KAAA;IAIJ,sBAAsB,yBAClB;KACE,aAAa;KACb,WAAW;IACb,IACA,KAAA;IAEJ,wBACE,sBAAsB,2BAClB;KACE,aAAa,QAAQ;KACrB,gBAAgB;KAChB,aAAa;KACb,eAAe;IACjB,IACA,KAAA;IAIN,sBACE,wBAAwB,UACpB;KACE,aAAa;KACb,iBAAiB;KACjB,aAAa;IACf,IACA,KAAA;IACN;GACF,GACA,UACF;GACA,YAAY;GACZ,kCAAkC;GAClC,aAAa,QAAQ,cAAc,IAAI;GACvC,aAAa,OAAO,aAAa,cAAc,UAAU;GACzD,aAAa,OAAO,WAAW,cAAc,SAAS;GACtD,mBAAmB;GACnB;EACF,SAAS,GAAG;GAcV,MAAM,QAAQ,QAAQ;GAEtB,eAAoB,KAAK;GACzB,gBAAgB,KAAA;GAChB,IAAI,SAAS,aAAa,iBAAiB;IACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;IAGhD,aAAa,QAAQ,IAAI;IAMzB,kBAAkB,QAAQ,IAAI;IAC9B;GACF;GACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;GAEF,MAAM;EACR;CACF;CAEA,OAAO,gBAAgB,kDAAkD;CACzE,OAAO,eAAe,oDAAoD;CAI1E,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,MAAM,oBAA8B,CAAC;CACrC,MAAM,EAAC,SAAS,iBAAiB,YAAW,SAAS;CACrD,IAAI,WAAW;EACb,MAAM,eAAe,YAAY,IAAI;EACrC,gBAAqB,WACnB,aAAa,OAAO,YAAY,YAAY,IAAI,IAAI,YAAY,CAClE;CACF;CACA,IAAI,CAAC,WACH,QAAQ;MACH;EACL,GAAG,OAAO,wBAAwB,SAAS;EAC3C,OAAO,WAAW,oBAAoB;EACtC,IAAI,kBACF,wBAAwB,QAAQ,IAAI;EAItC,MAAM,mBAAmB,UACtB,IACC,YACE,gBACA;GAAC,GAAG;GAAK,qCAAqC;EAAS,GACvD,QACF,GACA,mBACF,CAAC,CACA,gBAAgB,eAAe,QAAQ,CAAC;EAC3C,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;CACF;CAEA,IAAI,cAAc;CAClB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAc,cAAc,oBAAoB,CAAC,CAAC,WAAW;GAC3D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;GAClE,aAAa,OAAO,kBAAkB,OAAO;EAC/C,CAAC;CACH;CAOA,MAAM,gBAAgB,qBAClB,cACA,QAAQ,IAAI,CAAC,aAAa,eAAe,CAAC;CAK9C,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAGA,gBAAgB,qBAAqB,iBAAiB;EACtD;EACA;EACA,kBAdqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAaF,GACA,QACA,cACF;CAEA,cAAmB,WAAW;EAC5B,aAAa,MAAM;EACnB,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CASD,UAAU;CACV,MAAM,eACJ,IACA,QACA,yBACA,eACA,SACF;AACF;;;;;;;AAQA,eAAe,gBACb,IACA,eACA,OACqB;CACrB,MAAM,WAAW,MAAM,gBACrB,IACA,eACA,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC;CAC7D,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,OAAO;AACT;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,CAAC,CAAC,MAAM,OAAM,MAAK;CACjB,IAAI,CAAC,SACH,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CAEF,MAAM;AACR,CAAC,CACL"}

package/out/zero-cache/src/server/replicator.js

@@ -6,8 +6,8 @@ import { parentWorker, singleProcessMode } from "../types/processes.js";
 import { getShardConfig } from "../types/shards.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { Database } from "../../../zqlite/src/db.js";
-import { getOrCreateGauge } from "../observability/metrics.js";
 import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruption.js";
+import { getOrCreateGauge } from "../observability/metrics.js";
 import { SupportingProcesses, exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { installProfileHandler } from "../types/profiler.js";

package/out/zero-cache/src/server/syncer.js

@@ -6,11 +6,9 @@ import { parentWorker, singleProcessMode } from "../types/processes.js";
 import { getShardID } from "../types/shards.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { randInt } from "../../../shared/src/rand.js";
-import { getOrCreateGauge } from "../observability/metrics.js";
-import { Snapshotter } from "../services/view-syncer/snapshotter.js";
-import { PipelineDriver } from "../services/view-syncer/pipeline-driver.js";
 import { connectPgClient } from "../types/pg.js";
 import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruption.js";
+import { getOrCreateGauge } from "../observability/metrics.js";
 import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { installProfileHandler } from "../types/profiler.js";
@@ -28,6 +26,8 @@ import { PusherService } from "../services/mutagen/pusher.js";
 import { BatchPacer } from "../services/view-syncer/batch-pacer.js";
 import { ConnectionContextManagerImpl } from "../services/view-syncer/connection-context-manager.js";
 import { DeferredWritesBudget } from "../services/view-syncer/deferred-writes-budget.js";
+import { Snapshotter } from "../services/view-syncer/snapshotter.js";
+import { PipelineDriver } from "../services/view-syncer/pipeline-driver.js";
 import { QueryStats } from "../services/view-syncer/query-stats.js";
 import { SnapshotRowCache } from "../services/view-syncer/snapshot-row-cache.js";
 import { ViewSyncerService } from "../services/view-syncer/view-syncer.js";

package/out/zero-cache/src/services/analyze.js

@@ -5,10 +5,10 @@ import { Debug } from "../../../zql/src/builder/debug-delegate.js";
 import { Database } from "../../../zqlite/src/db.js";
 import { TableSource } from "../../../zqlite/src/table-source.js";
 import { computeZqlSpecs, mustGetTableSpec } from "../db/lite-tables.js";
+import _usingCtx from "../../../_virtual/_@oxc-project_runtime@0.154.0/helpers/esm/usingCtx.js";
 import { createSQLiteCostModel } from "../../../zqlite/src/sqlite-cost-model.js";
-import { MAX_ANALYZE_ROWS, runAst } from "./run-ast.js";
 import { explainQueries } from "../../../zqlite/src/explain-queries.js";
-import _usingCtx from "../../../_virtual/_@oxc-project_runtime@0.154.0/helpers/esm/usingCtx.js";
+import { MAX_ANALYZE_ROWS, runAst } from "./run-ast.js";
 import { TimeSliceTimer } from "./view-syncer/view-syncer.js";
 //#region ../zero-cache/src/services/analyze.ts
 var TIME_SLICE_LAP_THRESHOLD_MS = 200;

package/out/zero-cache/src/services/change-source/change-source.d.ts

@@ -19,6 +19,13 @@ export interface ChangeSource {
         firstCommitTimeMs: number;
         nextSendTimeMs: number;
     } | null> | null;
+    /**
+     * Requests that a lag report be initiated now rather than at the next
+     * report interval, e.g. so that a subscriber that has just caught up can
+     * determine its readiness promptly. Optional; a no-op if lag reporting is
+     * not supported / enabled, or if a report is already in flight.
+     */
+    requestLagReport?(): void;
     /**
      * Starts a stream of changes starting after the specific watermark,
      * with a corresponding sink for upstream acknowledgements.

package/out/zero-cache/src/services/change-source/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-source/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAE,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,KAAK,EACV,eAAe,EACf,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAErC;;;;OAIG;IACH,IAAI,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;CAClC,CAAC,CAAC,+EAA+E;AAElF,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,gBAAgB,IAAI,OAAO,CAAC;QAC1B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;IAEjB;;;OAGG;IACH,WAAW,CACT,cAAc,EAAE,MAAM,EACtB,gBAAgB,CAAC,EAAE,eAAe,EAAE,GACnC,OAAO,CAAC,YAAY,CAAC,CAAC;IAEzB;;OAEG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-source/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAE,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,KAAK,EACV,eAAe,EACf,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAErC;;;;OAIG;IACH,IAAI,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;CAClC,CAAC,CAAC,+EAA+E;AAElF,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,gBAAgB,IAAI,OAAO,CAAC;QAC1B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;IAEjB;;;;;OAKG;IACH,gBAAgB,CAAC,IAAI,IAAI,CAAC;IAE1B;;;OAGG;IACH,WAAW,CACT,cAAc,EAAE,MAAM,EACtB,gBAAgB,CAAC,EAAE,eAAe,EAAE,GACnC,OAAO,CAAC,YAAY,CAAC,CAAC;IAEzB;;OAEG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.js

@@ -3,8 +3,8 @@ import { AbortError } from "../../../../../shared/src/abort-error.js";
 import { must } from "../../../../../shared/src/must.js";
 import { promiseOrAbort } from "../../../../../shared/src/promise-race.js";
 import { stringify } from "../../../../../shared/src/bigint-json.js";
-import "../../../types/lite.js";
 import { randInt } from "../../../../../shared/src/rand.js";
+import "../../../types/lite.js";
 import { stateVersionFromString, stateVersionToString } from "../../../types/state-version.js";
 import { Queue } from "../../../../../shared/src/queue.js";
 import { CustomKeyMap } from "../../../../../shared/src/custom-key-map.js";

package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts

@@ -86,6 +86,31 @@ export type InitializeResult = {
      * litestream state (see {@link ResumableBackup}).
      */
     newBackupLineage: boolean;
+    /**
+     * How the replica was initialized, and the durations of its phases, for
+     * the replication-manager's start metrics (see `ServerStartMetrics`).
+     * Absent for change sources that do not report them.
+     */
+    startTimings?: InitializeTimings | undefined;
+};
+/**
+ * How a replica was initialized (see {@link InitializeResult.startTimings}):
+ * - `fork`: restored from the backup of a running replica, with a new slot
+ * - `resume`: restored from the backup of an orphaned replica, with its slot
+ * - `rmv1`: restored from the backup of the replica whose slot it takes over
+ * - `initial_sync`: initially synced
+ */
+export type InitializeKind = 'fork' | 'resume' | 'rmv1' | 'initial_sync';
+export type InitializeTimings = {
+    kind: InitializeKind;
+    /**
+     * The time spent selecting the replica to fork (or resume) and creating (or
+     * claiming) its slot, i.e. before restoring, including any wait for the
+     * source replica to pass the new slot's position.
+     */
+    forkWaitMs?: number | undefined;
+    /** The time spent restoring the replica from its backup. */
+    restoreMs?: number | undefined;
 };
 /**
  * Where the backup goes after a restore, when it can continue from the

package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replica-restore.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAEL,KAAK,kBAAkB,EAExB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAEnD,MAAM,WAAW,SAAS;IACxB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,yEAAyE;AACzE,MAAM,WAAW,qBAAsB,SAAQ,SAAS,EAAE,kBAAkB;IAC1E;;;OAGG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,CACnC,aAAa,CAAC,EAAE,MAAM,KACnB,OAAO,CAAC,qBAAqB,GAAG,IAAI,GAAG,aAAa,CAAC,CAAC;AAE3D,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,sBAAsB,GAAG,SAAS,CAAC;IACtD;;;;;OAKG;IACH,OAAO,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,YAAY,EAAE,YAAY,CAAC;IAC3B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB;;;;;;;;;OASG;IACH,0BAA0B,EAAE,OAAO,CAAC;IAEpC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE5C;;;;;;;;;;;OAWG;IACH,gBAAgB,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,CAAC;AAEhE;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAOF;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,WAAW,CAAC,EAAE,iBAAiB,GAAG,SAAS,GAC1C,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAkDtC"}
+{"version":3,"file":"replica-restore.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAEL,KAAK,kBAAkB,EAExB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAEnD,MAAM,WAAW,SAAS;IACxB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,yEAAyE;AACzE,MAAM,WAAW,qBAAsB,SAAQ,SAAS,EAAE,kBAAkB;IAC1E;;;OAGG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,CACnC,aAAa,CAAC,EAAE,MAAM,KACnB,OAAO,CAAC,qBAAqB,GAAG,IAAI,GAAG,aAAa,CAAC,CAAC;AAE3D,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,sBAAsB,GAAG,SAAS,CAAC;IACtD;;;;;OAKG;IACH,OAAO,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,YAAY,EAAE,YAAY,CAAC;IAC3B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB;;;;;;;;;OASG;IACH,0BAA0B,EAAE,OAAO,CAAC;IAEpC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE5C;;;;;;;;;;;OAWG;IACH,gBAAgB,EAAE,OAAO,CAAC;IAE1B;;;;OAIG;IACH,YAAY,CAAC,EAAE,iBAAiB,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,cAAc,CAAC;AAEzE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,cAAc,CAAC;IACrB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,CAAC;AAEhE;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAOF;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,WAAW,CAAC,EAAE,iBAAiB,GAAG,SAAS,GAC1C,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAkDtC"}

package/out/zero-cache/src/services/change-source/common/replica-restore.js.map

@@ -1 +1 @@
-{"version":3,"file":"replica-restore.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport type {LitestreamConfig} from '../../../config/normalize.ts';\nimport {\n  tryRestore,\n  type ReplicaConstraints,\n  type RestoreResult,\n} from '../../litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../../litestream/metrics.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource} from '../change-source.ts';\nimport type {InitCleanup} from './init-cleanup.ts';\n\nexport interface PurgeLock {\n  release(): Promise<void>;\n}\n\n/** A PurgeLock that also constrains the replica that can be restored. */\nexport interface ConstrainingPurgeLock extends PurgeLock, ReplicaConstraints {\n  /**\n   * The head of the change log when the lock was acquired, from which (or\n   * from past which) the change log is resumed.\n   */\n  readonly headWatermark: string;\n}\n\n/**\n * Acquires a purge lock on the PG change-log (if it is not empty).\n *\n * When resuming the change-log from a replication slot, `slotWatermark` is the\n * slot's position (as a watermark). If the change-log's head is behind it, the\n * change-log may be missing transactions that the slot will not stream, in\n * which case nothing is locked and `'behind-slot'` is returned.\n */\nexport type PgChangeLogPurgeLocker = (\n  slotWatermark?: string,\n) => Promise<ConstrainingPurgeLock | null | 'behind-slot'>;\n\nexport type RestoreOptions = {\n  litestream?: LitestreamConfig;\n  /**\n   * With the PG change-log enabled, purge-locks the change-log before the\n   * replica is restored, which constrains the replica to one from which the\n   * change-log can be resumed. The lock is acquired as late as possible (i.e.\n   * after a replication slot is created, if applicable) because, if the\n   * change-db is the upstream db, a transaction holding the lock would block\n   * the creation of a replication slot.\n   */\n  acquirePurgeLock?: PgChangeLogPurgeLocker | undefined;\n  /**\n   * Resources acquired by the initialization (e.g. a claimed replication slot)\n   * are registered here to be released if the initialization attempt fails.\n   * Note that a purge lock acquired with `acquirePurgeLock` must be registered\n   * here by the locker, so that it is released on failure.\n   */\n  cleanup?: InitCleanup | undefined;\n};\n\nexport type InitializeResult = {\n  subscriptionState: SubscriptionState;\n  changeSource: ChangeSource;\n  destinationBackupURL: string | undefined;\n  /**\n   * The replica this change stream belongs to, from the upstream `replicas`\n   * table. It is part of the identity the SQLite change log records, because a\n   * generation (i.e. `replicaVersion`) is shared by every sibling of a forked\n   * replica and so cannot distinguish two siblings' logs. `null` when the\n   * change source has no upstream table to identify itself from.\n   */\n  replicaID: string | null;\n\n  /**\n   * Whether server readiness should be gated on the first litestream backup,\n   * in order to ensure that a newly connecting subscriber can restore from\n   * this change-streamer's backup.\n   *\n   * This should be true when the backup destination differs from where the\n   * backup was restored from, which is the case for (1) initial-sync and\n   * (2) most initialization cases in RMv2, with the exception being when\n   * an abandoned replica is resumed.\n   */\n  waitForBackupBeforeServing: boolean;\n\n  /**\n   * Whether the PG change-log's head was behind the position of the\n   * replication slot from which replication resumes (see\n   * {@link PgChangeLogPurgeLocker}), in which case it is re-initialized from\n   * the restored replica if it does not contain the replica's changes.\n   */\n  pgChangeLogBehindSlot?: boolean | undefined;\n\n  /**\n   * Whether the litestream backup to `destinationBackupURL` starts a new\n   * (litestream v5) backup lineage, i.e. one that no local litestream state\n   * belongs to. Any such state (e.g. from a previous run of a restarted\n   * container, whose restore reused the existing replica) describes a\n   * different lineage and must be discarded before replicating, or litestream\n   * resolves its position from it and fails (and auto-recovers) on its first\n   * sync to the empty lineage.\n   *\n   * This is false when the backup continues a lineage from the local\n   * litestream state (see {@link ResumableBackup}).\n   */\n  newBackupLineage: boolean;\n};\n\n/**\n * Where the backup goes after a restore, when it can continue from the\n * restored replica's local litestream state instead of starting a new lineage\n * with a full snapshot:\n * - `fork`: a new, unique subfolder of `baseURL`, which the restore seeds with\n *   the restored backup files.\n */\nexport type BackupDestination = {type: 'fork'; baseURL: string};\n\n/**\n * A backup lineage at `backupPath` that the litestream backup continues from\n * the local litestream state and the restored replica's WAL.\n */\nexport type ResumableBackup = {\n  backupPath: string;\n};\n\n// A short retry is much cheaper than an initial Postgres sync, while keeping a\n// persistent failure from delaying startup appreciably.\nconst MAX_RESTORE_ATTEMPTS = 3;\nconst RESTORE_RETRY_DELAY_MS = 5_000;\n\n/**\n * Restores the replica from `config.backupURL`. With a `destination`, it\n * returns the backup that continues from the restored replica, if the\n * restore prepared one (e.g. it was not skipped because the replica already\n * exists).\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  destination?: BackupDestination | undefined,\n): Promise<ResumableBackup | undefined> {\n  const start = performance.now();\n  let result: RestoreResult | undefined;\n  try {\n    // `tryRestore` returns ordinary restore outcomes (no backup or an invalid\n    // replica) rather than throwing; preserve their existing immediate\n    // initial-sync fallback. Retry every thrown restore error twice before\n    // falling back to the initial Postgres sync.\n    for (let attemptNum = 1; attemptNum <= MAX_RESTORE_ATTEMPTS; attemptNum++) {\n      // Each attempt seeds a new subfolder, as a failed attempt can leave\n      // files in its subfolder.\n      const backupPath =\n        destination?.type === 'fork' ? String(Date.now()) : undefined;\n      try {\n        const attempt = await tryRestore(\n          lc,\n          config,\n          replicaFile,\n          replicaConstraints,\n          'replication_manager',\n          undefined,\n          backupPath\n            ? {url: new URL(backupPath, destination?.baseURL).toString()}\n            : undefined,\n        );\n        result = attempt.result;\n        return backupPath && attempt.forked ? {backupPath} : undefined;\n      } catch (e) {\n        if (attemptNum === MAX_RESTORE_ATTEMPTS) {\n          lc.error?.(\n            `litestream restore failed after ${attemptNum} attempts; resyncing the replica`,\n            e,\n          );\n          break;\n        }\n\n        lc.warn?.(\n          `litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`,\n          e,\n        );\n        await sleep(RESTORE_RETRY_DELAY_MS);\n      }\n    }\n    return undefined;\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(config, 'replication_manager');\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n"],"mappings":";;;;AAiIA,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;;;;;;;AAQ/B,eAAsB,eACpB,IACA,QACA,aACA,oBACA,aACsC;CACtC,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;EAKF,KAAK,IAAI,aAAa,GAAG,cAAc,sBAAsB,cAAc;GAGzE,MAAM,aACJ,aAAa,SAAS,SAAS,OAAO,KAAK,IAAI,CAAC,IAAI,KAAA;GACtD,IAAI;IACF,MAAM,UAAU,MAAM,WACpB,IACA,QACA,aACA,oBACA,uBACA,KAAA,GACA,aACI,EAAC,KAAK,IAAI,IAAI,YAAY,aAAa,OAAO,CAAC,CAAC,SAAS,EAAC,IAC1D,KAAA,CACN;IACA,SAAS,QAAQ;IACjB,OAAO,cAAc,QAAQ,SAAS,EAAC,WAAU,IAAI,KAAA;GACvD,SAAS,GAAG;IACV,IAAI,eAAe,sBAAsB;KACvC,GAAG,QACD,mCAAmC,WAAW,mCAC9C,CACF;KACA;IACF;IAEA,GAAG,OACD,8BAA8B,WAAW,uBAAuB,uBAAuB,cAAc,aAAa,EAAE,MAAM,qBAAqB,IAC/I,CACF;IACA,MAAM,MAAM,sBAAsB;GACpC;EACF;EACA;CACF,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,qBAChC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,CAAC,CAAC,IAAI,GAAG,MAAM;EACrC,0BAA0B,CAAC,CAAC,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF"}
+{"version":3,"file":"replica-restore.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport type {LitestreamConfig} from '../../../config/normalize.ts';\nimport {\n  tryRestore,\n  type ReplicaConstraints,\n  type RestoreResult,\n} from '../../litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../../litestream/metrics.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource} from '../change-source.ts';\nimport type {InitCleanup} from './init-cleanup.ts';\n\nexport interface PurgeLock {\n  release(): Promise<void>;\n}\n\n/** A PurgeLock that also constrains the replica that can be restored. */\nexport interface ConstrainingPurgeLock extends PurgeLock, ReplicaConstraints {\n  /**\n   * The head of the change log when the lock was acquired, from which (or\n   * from past which) the change log is resumed.\n   */\n  readonly headWatermark: string;\n}\n\n/**\n * Acquires a purge lock on the PG change-log (if it is not empty).\n *\n * When resuming the change-log from a replication slot, `slotWatermark` is the\n * slot's position (as a watermark). If the change-log's head is behind it, the\n * change-log may be missing transactions that the slot will not stream, in\n * which case nothing is locked and `'behind-slot'` is returned.\n */\nexport type PgChangeLogPurgeLocker = (\n  slotWatermark?: string,\n) => Promise<ConstrainingPurgeLock | null | 'behind-slot'>;\n\nexport type RestoreOptions = {\n  litestream?: LitestreamConfig;\n  /**\n   * With the PG change-log enabled, purge-locks the change-log before the\n   * replica is restored, which constrains the replica to one from which the\n   * change-log can be resumed. The lock is acquired as late as possible (i.e.\n   * after a replication slot is created, if applicable) because, if the\n   * change-db is the upstream db, a transaction holding the lock would block\n   * the creation of a replication slot.\n   */\n  acquirePurgeLock?: PgChangeLogPurgeLocker | undefined;\n  /**\n   * Resources acquired by the initialization (e.g. a claimed replication slot)\n   * are registered here to be released if the initialization attempt fails.\n   * Note that a purge lock acquired with `acquirePurgeLock` must be registered\n   * here by the locker, so that it is released on failure.\n   */\n  cleanup?: InitCleanup | undefined;\n};\n\nexport type InitializeResult = {\n  subscriptionState: SubscriptionState;\n  changeSource: ChangeSource;\n  destinationBackupURL: string | undefined;\n  /**\n   * The replica this change stream belongs to, from the upstream `replicas`\n   * table. It is part of the identity the SQLite change log records, because a\n   * generation (i.e. `replicaVersion`) is shared by every sibling of a forked\n   * replica and so cannot distinguish two siblings' logs. `null` when the\n   * change source has no upstream table to identify itself from.\n   */\n  replicaID: string | null;\n\n  /**\n   * Whether server readiness should be gated on the first litestream backup,\n   * in order to ensure that a newly connecting subscriber can restore from\n   * this change-streamer's backup.\n   *\n   * This should be true when the backup destination differs from where the\n   * backup was restored from, which is the case for (1) initial-sync and\n   * (2) most initialization cases in RMv2, with the exception being when\n   * an abandoned replica is resumed.\n   */\n  waitForBackupBeforeServing: boolean;\n\n  /**\n   * Whether the PG change-log's head was behind the position of the\n   * replication slot from which replication resumes (see\n   * {@link PgChangeLogPurgeLocker}), in which case it is re-initialized from\n   * the restored replica if it does not contain the replica's changes.\n   */\n  pgChangeLogBehindSlot?: boolean | undefined;\n\n  /**\n   * Whether the litestream backup to `destinationBackupURL` starts a new\n   * (litestream v5) backup lineage, i.e. one that no local litestream state\n   * belongs to. Any such state (e.g. from a previous run of a restarted\n   * container, whose restore reused the existing replica) describes a\n   * different lineage and must be discarded before replicating, or litestream\n   * resolves its position from it and fails (and auto-recovers) on its first\n   * sync to the empty lineage.\n   *\n   * This is false when the backup continues a lineage from the local\n   * litestream state (see {@link ResumableBackup}).\n   */\n  newBackupLineage: boolean;\n\n  /**\n   * How the replica was initialized, and the durations of its phases, for\n   * the replication-manager's start metrics (see `ServerStartMetrics`).\n   * Absent for change sources that do not report them.\n   */\n  startTimings?: InitializeTimings | undefined;\n};\n\n/**\n * How a replica was initialized (see {@link InitializeResult.startTimings}):\n * - `fork`: restored from the backup of a running replica, with a new slot\n * - `resume`: restored from the backup of an orphaned replica, with its slot\n * - `rmv1`: restored from the backup of the replica whose slot it takes over\n * - `initial_sync`: initially synced\n */\nexport type InitializeKind = 'fork' | 'resume' | 'rmv1' | 'initial_sync';\n\nexport type InitializeTimings = {\n  kind: InitializeKind;\n  /**\n   * The time spent selecting the replica to fork (or resume) and creating (or\n   * claiming) its slot, i.e. before restoring, including any wait for the\n   * source replica to pass the new slot's position.\n   */\n  forkWaitMs?: number | undefined;\n  /** The time spent restoring the replica from its backup. */\n  restoreMs?: number | undefined;\n};\n\n/**\n * Where the backup goes after a restore, when it can continue from the\n * restored replica's local litestream state instead of starting a new lineage\n * with a full snapshot:\n * - `fork`: a new, unique subfolder of `baseURL`, which the restore seeds with\n *   the restored backup files.\n */\nexport type BackupDestination = {type: 'fork'; baseURL: string};\n\n/**\n * A backup lineage at `backupPath` that the litestream backup continues from\n * the local litestream state and the restored replica's WAL.\n */\nexport type ResumableBackup = {\n  backupPath: string;\n};\n\n// A short retry is much cheaper than an initial Postgres sync, while keeping a\n// persistent failure from delaying startup appreciably.\nconst MAX_RESTORE_ATTEMPTS = 3;\nconst RESTORE_RETRY_DELAY_MS = 5_000;\n\n/**\n * Restores the replica from `config.backupURL`. With a `destination`, it\n * returns the backup that continues from the restored replica, if the\n * restore prepared one (e.g. it was not skipped because the replica already\n * exists).\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  destination?: BackupDestination | undefined,\n): Promise<ResumableBackup | undefined> {\n  const start = performance.now();\n  let result: RestoreResult | undefined;\n  try {\n    // `tryRestore` returns ordinary restore outcomes (no backup or an invalid\n    // replica) rather than throwing; preserve their existing immediate\n    // initial-sync fallback. Retry every thrown restore error twice before\n    // falling back to the initial Postgres sync.\n    for (let attemptNum = 1; attemptNum <= MAX_RESTORE_ATTEMPTS; attemptNum++) {\n      // Each attempt seeds a new subfolder, as a failed attempt can leave\n      // files in its subfolder.\n      const backupPath =\n        destination?.type === 'fork' ? String(Date.now()) : undefined;\n      try {\n        const attempt = await tryRestore(\n          lc,\n          config,\n          replicaFile,\n          replicaConstraints,\n          'replication_manager',\n          undefined,\n          backupPath\n            ? {url: new URL(backupPath, destination?.baseURL).toString()}\n            : undefined,\n        );\n        result = attempt.result;\n        return backupPath && attempt.forked ? {backupPath} : undefined;\n      } catch (e) {\n        if (attemptNum === MAX_RESTORE_ATTEMPTS) {\n          lc.error?.(\n            `litestream restore failed after ${attemptNum} attempts; resyncing the replica`,\n            e,\n          );\n          break;\n        }\n\n        lc.warn?.(\n          `litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`,\n          e,\n        );\n        await sleep(RESTORE_RETRY_DELAY_MS);\n      }\n    }\n    return undefined;\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(config, 'replication_manager');\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n"],"mappings":";;;;AA6JA,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;;;;;;;AAQ/B,eAAsB,eACpB,IACA,QACA,aACA,oBACA,aACsC;CACtC,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;EAKF,KAAK,IAAI,aAAa,GAAG,cAAc,sBAAsB,cAAc;GAGzE,MAAM,aACJ,aAAa,SAAS,SAAS,OAAO,KAAK,IAAI,CAAC,IAAI,KAAA;GACtD,IAAI;IACF,MAAM,UAAU,MAAM,WACpB,IACA,QACA,aACA,oBACA,uBACA,KAAA,GACA,aACI,EAAC,KAAK,IAAI,IAAI,YAAY,aAAa,OAAO,CAAC,CAAC,SAAS,EAAC,IAC1D,KAAA,CACN;IACA,SAAS,QAAQ;IACjB,OAAO,cAAc,QAAQ,SAAS,EAAC,WAAU,IAAI,KAAA;GACvD,SAAS,GAAG;IACV,IAAI,eAAe,sBAAsB;KACvC,GAAG,QACD,mCAAmC,WAAW,mCAC9C,CACF;KACA;IACF;IAEA,GAAG,OACD,8BAA8B,WAAW,uBAAuB,uBAAuB,cAAc,aAAa,EAAE,MAAM,qBAAqB,IAC/I,CACF;IACA,MAAM,MAAM,sBAAsB;GACpC;EACF;EACA;CACF,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,qBAChC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,CAAC,CAAC,IAAI,GAAG,MAAM;EACrC,0BAA0B,CAAC,CAAC,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF"}

package/out/zero-cache/src/services/change-source/common/replica-schema.js

@@ -1,8 +1,8 @@
+import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../../db/sqlite-corruption.js";
 import { populateFromExistingTables } from "../../replicator/schema/column-metadata.js";
 import { listTables } from "../../../db/lite-tables.js";
 import { BACKFILLING_TABLE, populateBackfillingFromColumnMetadata } from "../../replicator/schema/backfilling.js";
 import { CREATE_RUNTIME_EVENTS_TABLE, recordEvent } from "../../replicator/schema/replication-state.js";
-import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../../db/sqlite-corruption.js";
 import { deleteLiteDB } from "../../../db/delete-lite-db.js";
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
 import { runSchemaMigrations } from "../../../db/migration-lite.js";

package/out/zero-cache/src/services/change-source/custom/change-source.js

@@ -5,8 +5,8 @@ import { changeStreamMessageSchema } from "../protocol/current/downstream.js";
 import "../protocol/current/upstream.js";
 import { Database } from "../../../../../zqlite/src/db.js";
 import { computeZqlSpecs } from "../../../db/lite-tables.js";
-import { createReplicationStateTables, getSubscriptionState, initReplicationState } from "../../replicator/schema/replication-state.js";
 import { StatementRunner } from "../../../db/statements.js";
+import { createReplicationStateTables, getSubscriptionState, initReplicationState } from "../../replicator/schema/replication-state.js";
 import { stream } from "../../../types/streams.js";
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
 import { ChangeProcessor } from "../../replicator/change-processor.js";

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js

@@ -3,11 +3,11 @@ import { parse } from "../../../../../shared/src/valita.js";
 import { must } from "../../../../../shared/src/must.js";
 import { randomCharacters } from "../../../../../shared/src/random-values.js";
 import { equals } from "../../../../../shared/src/set-utils.js";
-import { id } from "../../../types/sql.js";
 import { connectPgClient, pgClient } from "../../../types/pg.js";
 import { toStateVersionString } from "./lsn.js";
 import { READONLY } from "../../../db/mode-enum.js";
 import { getPublicationInfo } from "./schema/published.js";
+import { id } from "../../../types/sql.js";
 import { Queue } from "../../../../../shared/src/queue.js";
 import { resumePoint } from "../protocol/backfill-progress.js";
 import { SchemaIncompatibilityError } from "../common/backfill-manager.js";

package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts

@@ -51,7 +51,14 @@ export declare function initializePostgresChangeSource(lc: LogContext, upstreamU
  *   log), there is no floor.
  */
 export declare function minStartWatermark(replicaWatermark: string, initialSynced: boolean, slotWatermark: string | undefined, purgeLock: ConstrainingPurgeLock | null): string | undefined;
-export declare function getSourceAndDestinationReplicas(lc: LogContext, sql: PostgresDB, shard: ShardID, epoch: number, slotFailover: boolean, gracePeriodMs: number, pollIntervalMs?: number): Promise<{
+export declare function getSourceAndDestinationReplicas(lc: LogContext, sql: PostgresDB, shard: ShardID, epoch: number, slotFailover: boolean, gracePeriodMs: number, 
+/**
+ * Whether a fork waits for the source replica's slot to pass a transaction
+ * after the destination slot's position (rather than just the position),
+ * which is necessary for the PG change-log to be resumable from the
+ * destination slot (see forkOrResumeReplica()).
+ */
+waitForStoredTxPastSlot: boolean, pollIntervalMs?: number): Promise<{
     restoreFrom: ReplicaState;
     replicateTo: ReplicaState;
     /** The position of the slot from which replication resumes. */

package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAOxE,OAAO,EAEL,KAAK,qBAAqB,EAC1B,KAAK,gBAAgB,EACrB,KAAK,cAAc,EAEpB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAiC,KAAK,GAAG,EAAC,MAAM,UAAU,CAAC;AASlE,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG;IAG/C,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,4BAA4B,EAAE,MAAM,CAAC;IAErC;;;;;OAKG;IACH,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EACE,KAAK,EACL,cAAc,EACd,QAAQ,EACR,4BAA4B,EAC5B,WAAW,GACZ,GAAE,iBAMF,EACD,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CA4I3B;AA6KD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,wBAAgB,iBAAiB,CAC/B,gBAAgB,EAAE,MAAM,EACxB,aAAa,EAAE,OAAO,EACtB,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,SAAS,EAAE,qBAAqB,GAAG,IAAI,GACtC,MAAM,GAAG,SAAS,CAQpB;AAsBD,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,OAAO,EACrB,aAAa,EAAE,MAAM,EACrB,cAAc,SAA2B,GACxC,OAAO,CACN;IACE,WAAW,EAAE,YAAY,CAAC;IAC1B,WAAW,EAAE,YAAY,CAAC;IAC1B,+DAA+D;IAC/D,OAAO,EAAE,GAAG,CAAC;IACb;;;OAGG;IACH,WAAW,EAAE,QAAQ,CAAC;CACvB,GACD,SAAS,CACZ,CA+LA"}
+{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAOxE,OAAO,EAEL,KAAK,qBAAqB,EAC1B,KAAK,gBAAgB,EAErB,KAAK,cAAc,EAEpB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAiC,KAAK,GAAG,EAAC,MAAM,UAAU,CAAC;AASlE,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG;IAG/C,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,4BAA4B,EAAE,MAAM,CAAC;IAErC;;;;;OAKG;IACH,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EACE,KAAK,EACL,cAAc,EACd,QAAQ,EACR,4BAA4B,EAC5B,WAAW,GACZ,GAAE,iBAMF,EACD,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CAgJ3B;AAkMD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,wBAAgB,iBAAiB,CAC/B,gBAAgB,EAAE,MAAM,EACxB,aAAa,EAAE,OAAO,EACtB,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,SAAS,EAAE,qBAAqB,GAAG,IAAI,GACtC,MAAM,GAAG,SAAS,CAQpB;AAsBD,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,OAAO,EACrB,aAAa,EAAE,MAAM;AACrB;;;;;GAKG;AACH,uBAAuB,EAAE,OAAO,EAChC,cAAc,SAA2B,GACxC,OAAO,CACN;IACE,WAAW,EAAE,YAAY,CAAC;IAC1B,WAAW,EAAE,YAAY,CAAC;IAC1B,+DAA+D;IAC/D,OAAO,EAAE,GAAG,CAAC;IACb;;;OAGG;IACH,WAAW,EAAE,QAAQ,CAAC;CACvB,GACD,SAAS,CACZ,CA2MA"}