@rocicorp/zero 1.11.0-canary.3 → 1.11.0-canary.30

package/out/zero-cache/src/services/change-source/pg/change-source.js

@@ -7,113 +7,30 @@ import { mapValues } from "../../../../../shared/src/objects.js";
 import { equals, intersection, symmetricDifferences } from "../../../../../shared/src/set-utils.js";
 import { stringify } from "../../../../../shared/src/bigint-json.js";
 import { upstreamSchema } from "../../../types/shards.js";
-import { Database } from "../../../../../zqlite/src/db.js";
 import { UnsupportedColumnDefaultError, mapPostgresToLiteColumn } from "../../../db/pg-to-lite.js";
 import { getOrCreateCounter } from "../../../observability/metrics.js";
-import { getSubscriptionStateAndContext } from "../../replicator/schema/replication-state.js";
-import { StatementRunner } from "../../../db/statements.js";
-import { connectPgClient, isPostgresError, pgClient } from "../../../types/pg.js";
+import { isPostgresError, pgClient } from "../../../types/pg.js";
+import { UnrecoverableError } from "../../running-state.js";
 import { majorVersionFromString, majorVersionToString } from "../../../types/state-version.js";
 import { fromBigInt, toBigInt, toStateVersionString } from "./lsn.js";
 import { getPublicationInfo, warnForSkippedIndexes } from "./schema/published.js";
 import { replicationEventSchema } from "./schema/ddl.js";
+import "./schema/replica-stage-enum.js";
 import { validate } from "./schema/validation.js";
-import { dropShard, getActiveReplicas, getInternalShardConfig, getReplicaAtVersion, internalPublicationPrefix, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotPrefix } from "./schema/shard.js";
-import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
+import { getInternalShardConfig, getReplicaAtVersion, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotPrefix } from "./schema/shard.js";
 import { mapIndexPredicateColumns } from "../../../db/index-predicate.js";
-import { restoreReplica } from "../common/replica-restore.js";
-import { initReplica } from "../common/replica-schema.js";
 import "../../../types/pg-versions.js";
 import { BackfillManager } from "../common/backfill-manager.js";
 import { ChangeStreamMultiplexer } from "../common/change-stream-multiplexer.js";
 import { subscribe } from "./logical-replication/stream.js";
-import { dropOldReplicasAndSlots } from "./replication-slots.js";
-import { ensureShardSchema } from "./schema/init.js";
-import { initialSync } from "./initial-sync.js";
+import "./initial-sync.js";
 import { streamBackfill } from "./backfill-stream.js";
+import { ReplicationSlotCleanupMonitor } from "./replication-slot-cleanup-monitor.js";
 import { registerReplicationSlotHealthMetrics } from "./replication-slot-health.js";
 import { nanoid } from "nanoid";
 import postgres from "postgres";
 import { PG_ADMIN_SHUTDOWN, PG_INSUFFICIENT_PRIVILEGE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/change-source.ts
-var REPLICA_SLOT_CLEANUP_INTERVAL_MS = 3e4;
-/**
-* Initializes a Postgres change source, including the initial sync of the
-* replica, before streaming changes from the corresponding logical replication
-* stream.
-*/
-async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbFile, syncOptions, context, lagReportIntervalMs = 0, restoreOptions = {}, { backupV5 } = { backupV5: true }, purgeLock, streamInboundTimeoutMs) {
-	const db = await connectPgClient(lc, upstreamURI, "change-source-init");
-	try {
-		await ensureShardSchema(lc, db, shard, syncOptions.installPartialIndexTriggers);
-		const restoredReplica = await selectAndRestoreReplica(lc, db, shard, replicaDbFile, restoreOptions);
-		let initialSyncedReplica;
-		await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, async (log, tx) => {
-			purgeLock?.release();
-			initialSyncedReplica = await initialSync(log, shard, tx, upstreamURI, syncOptions, context, { backupV5 });
-		});
-		const replica = new Database(lc, replicaDbFile);
-		const subscriptionState = getSubscriptionStateAndContext(new StatementRunner(replica));
-		replica.close();
-		const upstreamReplica = await checkAndUpdateUpstream(lc, db, shard, subscriptionState, (initialSyncedReplica ?? restoredReplica)?.id);
-		const backupPath = initialSyncedReplica ? initialSyncedReplica.backupPath : backupV5 ? String(Date.now()) : restoredReplica?.backupPath ?? null;
-		return {
-			subscriptionState,
-			changeSource: new PostgresChangeSource(lc, upstreamURI, shard, upstreamReplica, {
-				backupPath,
-				backupV5
-			}, context, lagReportIntervalMs, syncOptions.textCopy, streamInboundTimeoutMs),
-			destinationBackupURL: backupPath && restoreOptions.litestream?.backupURL ? new URL(backupPath, restoreOptions.litestream.backupURL).toString() : restoreOptions.litestream?.backupURL,
-			replicaID: upstreamReplica.id,
-			waitForBackupBeforeServing: initialSyncedReplica !== void 0 || backupPath !== (restoredReplica?.backupPath ?? null)
-		};
-	} finally {
-		await db.end();
-	}
-}
-async function selectAndRestoreReplica(lc, sql, shard, replicaFile, { litestream, constraints }) {
-	const replicas = (await getActiveReplicas(lc, sql, shard)).filter(({ generation }) => generation === (constraints?.replicaVersion ?? generation));
-	if (replicas.length === 0) {
-		lc.info?.(`no suitable replicas to restore from`, { replicas });
-		return;
-	}
-	const [replica] = replicas;
-	if (litestream?.backupURL) {
-		const { backupURL: backupBaseURL } = litestream;
-		const { slot, backupPath, confirmedFlushLsn } = replica;
-		const backupURL = new URL(backupPath ?? "", backupBaseURL).toString();
-		lc.info?.(`restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`, { replicas });
-		await restoreReplica(lc, {
-			...litestream,
-			backupURL
-		}, replicaFile, constraints);
-	}
-	return replica;
-}
-async function checkAndUpdateUpstream(lc, sql, shard, { replicaVersion, publications: subscribed, initialSyncContext }, replicaID) {
-	const upstreamReplica = await getReplicaAtVersion(lc, sql, shard, replicaVersion, replicaID, initialSyncContext);
-	if (!upstreamReplica) throw new AutoResetSignal(`No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`);
-	const requested = shard.publications.toSorted();
-	const replicated = upstreamReplica.publications.filter((p) => !p.startsWith(internalPublicationPrefix(shard))).sort();
-	if (!deepEqual(requested, replicated)) {
-		lc.warn?.(`Dropping shard to change publications to: [${requested}]`);
-		await sql.unsafe(dropShard(shard.appID, shard.shardNum));
-		throw new AutoResetSignal(`Requested publications [${requested}] do not match configured publications: [${replicated}]`);
-	}
-	if (!deepEqual(upstreamReplica.publications, subscribed)) throw new AutoResetSignal(`Upstream publications [${upstreamReplica.publications}] do not match subscribed publications [${subscribed}]`);
-	const exists = await sql`
-    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};
-  `.values();
-	if (exists.length !== subscribed.length) throw new AutoResetSignal(`Upstream publications [${exists.flat()}] do not contain all subscribed publications [${subscribed}]`);
-	const { slot } = upstreamReplica;
-	const result = await sql`
-    SELECT restart_lsn as "restartLSN", wal_status as "walStatus" FROM pg_replication_slots
-      WHERE slot_name = ${slot}`;
-	if (result.length === 0) throw new AutoResetSignal(`replication slot ${slot} is missing`);
-	const [{ restartLSN, walStatus }] = result;
-	if (restartLSN === null || walStatus === "lost") throw new AutoResetSignal(`replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`);
-	return upstreamReplica;
-}
 var MAX_LOW_PRIORITY_DELAY_MS = 1e3;
 /**
 * Postgres implementation of a {@link ChangeSource} backed by a logical
@@ -125,36 +42,39 @@ var PostgresChangeSource = class {
 	#upstreamUri;
 	#shard;
 	#replica;
+	#slotCleanupMonitor;
 	#backupOptions;
 	#context;
 	#lagReporter;
 	#textCopy;
 	#streamInboundTimeoutMs;
+	#minStartWatermark;
 	#subscribe;
 	#streamBackfill;
 	#stopped = false;
-	constructor(lc, upstreamUri, shard, replica, backupOptions, context, lagReportIntervalMs, textCopy, streamInboundTimeoutMs, deps = {}) {
+	constructor(lc, upstreamUri, shard, replica, pgVersion, backupOptions, context, lagReportIntervalMs, textCopy, streamInboundTimeoutMs, minStartWatermark, deps = {}) {
 		this.#lc = lc.withContext("component", "change-source");
 		this.#subscribe = deps.subscribe ?? subscribe;
 		this.#streamBackfill = deps.streamBackfill ?? streamBackfill;
 		this.#db = pgClient(lc, upstreamUri, "replication-monitor", {
-			max: 1,
-			["idle_timeout"]: 60
+			max: 3,
+			idle_timeout: 60
 		});
 		this.#upstreamUri = upstreamUri;
 		this.#shard = shard;
 		this.#replica = replica;
+		this.#slotCleanupMonitor = new ReplicationSlotCleanupMonitor(lc.withContext("component", "replication-slot-monitor"), shard, this.#db, pgVersion, replica.slot);
 		this.#backupOptions = backupOptions;
 		this.#context = context;
 		this.#textCopy = textCopy ?? false;
 		this.#streamInboundTimeoutMs = streamInboundTimeoutMs;
-		this.#lagReporter = lagReportIntervalMs > 0 ? new LagReporter(lc.withContext("component", "lag-reporter"), shard, this.#db, lagReportIntervalMs) : null;
+		this.#minStartWatermark = minStartWatermark;
+		this.#lagReporter = lagReportIntervalMs > 0 ? new LagReporter(lc.withContext("component", "lag-reporter"), shard, this.#db, pgVersion, lagReportIntervalMs) : null;
 		registerReplicationSlotHealthMetrics(this.#lc, this.#db, replica.slot, () => this.#stopped);
 	}
 	async stop() {
 		this.#stopped = true;
 		this.#lagReporter?.stop();
-		clearTimeout(this.#cleanupTimer);
 		await this.#db.end();
 	}
 	async startLagReporter() {
@@ -173,24 +93,32 @@ var PostgresChangeSource = class {
 		const config = await getInternalShardConfig(this.#db, this.#shard);
 		const { slot } = this.#replica;
 		this.#lc.info?.(`starting replication stream@${slot}`);
-		return this.startStreamInternal(slot, clientWatermark, config, backfillRequests);
+		await this.#checkConfirmedFlushLSN(slot, clientWatermark);
+		const changeStream = await this.startStreamInternal(slot, clientWatermark, config, backfillRequests);
+		const { signal } = changeStream.changes;
+		if (!signal.aborted) {
+			this.#slotCleanupMonitor.start();
+			signal.addEventListener("abort", () => this.#slotCleanupMonitor.stop());
+		}
+		return changeStream;
 	}
 	async startStreamInternal(slot, clientWatermark, shardConfig, backfillRequests) {
+		if (this.#minStartWatermark !== void 0 && majorVersionFromString(clientWatermark) < majorVersionFromString(this.#minStartWatermark)) throw new UnrecoverableError(`cannot start replication stream@${slot} from ${clientWatermark}, which is before its minimum start watermark ${this.#minStartWatermark}: transactions in between would be silently skipped`);
 		const clientStart = majorVersionFromString(clientWatermark) + 1n;
 		const { messages, acks } = await this.#subscribe(this.#lc, this.#db, slot, [...shardConfig.publications], clientStart, void 0, void 0, this.#streamInboundTimeoutMs);
-		const acker = new Acker(acks);
+		const acker = new Acker(acks, clientWatermark);
 		const changes = new ChangeStreamMultiplexer(this.#lc, clientWatermark);
 		const backfillManager = new BackfillManager(this.#lc, changes, (req) => this.#streamBackfill(this.#lc, this.#upstreamUri, this.#replica, req, { textCopy: this.#textCopy }));
 		changes.addProducers(messages, backfillManager).addListeners(backfillManager, acker);
 		backfillManager.run(clientWatermark, backfillRequests);
-		const changeMaker = new ChangeMaker(this.#shard, shardConfig, this.#db, this.#replica.initialSchema);
+		const changeMaker = new ChangeMaker(this.#shard, shardConfig, this.#db, this.#replica);
 		/**
 		* Determines if the incoming message is transactional, otherwise handling
 		* non-transactional messages with a downstream status message.
 		*/
 		const isTransactionalMessage = (lsn, msg) => {
 			if (msg.tag === "message" && msg.prefix === this.#lagReporter?.messagePrefix) {
-				changes.pushStatus(this.#lagReporter.processLagReportMessage(msg));
+				changes.pushStatus(this.#lagReporter.processLagReportMessage(msg, lsn));
 				return false;
 			}
 			const status = this.#lagReporter?.checkCurrentLSN(lsn);
@@ -256,6 +184,31 @@ var PostgresChangeSource = class {
 			acks: { push: (status) => acker.ack(status[2].watermark) }
 		};
 	}
+	/**
+	* Warns if the stream is about to start from before the slot's
+	* `confirmed_flush_lsn`, from which Postgres starts streaming instead.
+	*
+	* This is benign if the slot was only acked past `clientWatermark` for
+	* keepalives (i.e. with no transactions in between), which the
+	* {@link Acker} does on a caught-up stream. It is therefore not an error
+	* (`minStartWatermark` is the bound that is), but it is the evidence needed
+	* to diagnose a skipped interval.
+	*/
+	async #checkConfirmedFlushLSN(slot, clientWatermark) {
+		try {
+			const [row] = await this.#db`
+        SELECT confirmed_flush_lsn AS "confirmedFlushLSN"
+          FROM pg_replication_slots WHERE slot_name = ${slot}`;
+			const confirmed = row?.confirmedFlushLSN;
+			if (!confirmed) return;
+			const start = majorVersionFromString(clientWatermark);
+			const confirmedLSN = toBigInt(confirmed);
+			if (start < confirmedLSN) this.#lc.warn?.(`starting replication stream@${slot} from ${clientWatermark}, which is ${confirmedLSN - start} bytes before the slot's confirmed_flush_lsn ${confirmed} (${toStateVersionString(confirmed)}). Postgres starts from the latter, which skips nothing only if the slot was acked past ${clientWatermark} for keepalives.`);
+			else this.#lc.info?.(`replication slot ${slot} confirmed_flush_lsn: ${confirmed} (${toStateVersionString(confirmed)})`);
+		} catch (e) {
+			this.#lc.warn?.(`unable to read the position of slot ${slot}`, e);
+		}
+	}
 	async #logCurrentReplicaInfo() {
 		try {
 			const replica = await getReplicaAtVersion(this.#lc, this.#db, this.#shard, this.#replica.generation, this.#replica.id);
@@ -284,6 +237,13 @@ var PostgresChangeSource = class {
 		const sql = this.#db;
 		const { id: replicaID, slot } = this.#replica;
 		const replicasTable = `${upstreamSchema(this.#shard)}.replicas`;
+		await sql`
+      UPDATE ${sql(replicasTable)} 
+        SET "stage" = ${1},
+            "subscriberContext" = ${this.#context},
+            "backupPath" = ${this.#backupOptions.backupPath},
+            "backupV5" = ${this.#backupOptions.backupV5}
+        WHERE id = ${replicaID}`;
 		const result = await sql`
       SELECT pg_terminate_backend(active_pid) as terminated, active_pid as pid
         FROM pg_replication_slots
@@ -306,35 +266,22 @@ var PostgresChangeSource = class {
 			throw new AbortError(`replication slot ${slot} is missing. A different replication-manager should now be running on a new replication slot.`);
 		}
 		this.#lc.info?.(`terminated replication slots: ${JSON.stringify(result)}`);
-		await sql`
-      UPDATE ${sql(replicasTable)} 
-        SET "subscriberContext" = ${this.#context},
-            "backupPath" = ${this.#backupOptions.backupPath},
-            "backupV5" = ${this.#backupOptions.backupV5}
-        WHERE id = ${replicaID}`;
-		this.#cleanUpOlderReplicasAndSlots();
-	}
-	#cleanupTimer;
-	async #cleanUpOlderReplicasAndSlots() {
-		clearTimeout(this.#cleanupTimer);
-		try {
-			const result = await dropOldReplicasAndSlots(this.#lc, this.#db, this.#shard, this.#replica.rank);
-			if (result.draining === 0) {
-				this.#lc.info?.(`finished cleaning up replicas and slots`, { result });
-				return;
-			}
-			this.#lc.info?.(`old slots still draining`, { result });
-		} catch (e) {
-			this.#lc.warn?.(`error dropping replication slots`, e);
-		}
-		this.#cleanupTimer = setTimeout(() => this.#cleanUpOlderReplicasAndSlots(), REPLICA_SLOT_CLEANUP_INTERVAL_MS);
 	}
 };
 var Acker = class {
 	#acks;
-	#waitingForDownstreamAck = null;
-	constructor(acks) {
+	#waitingForDownstreamAck;
+	/**
+	* @param resumeWatermark the watermark the stream resumes after. What came
+	*     before it was received on an earlier connection, and only the
+	*     change-streamer knows whether it has persisted it, so keepalives are
+	*     not acked until the change-streamer has acked it. A change-streamer
+	*     whose SQLite change log is ahead of its backup would otherwise have
+	*     the slot moved past transactions that only that log holds.
+	*/
+	constructor(acks, resumeWatermark) {
 		this.#acks = acks;
+		this.#waitingForDownstreamAck = resumeWatermark;
 	}
 	onChange(change) {
 		switch (change[0]) {
@@ -373,33 +320,28 @@ var LagReporter = class LagReporter {
 	#lc;
 	messagePrefix;
 	#db;
-	#lagIntervalMs;
+	#reportIntervalMs;
+	#reportIntervalJitterMs;
 	#lagReportRetries = getOrCreateCounter("replication", "lag_report_retries", {
 		description: "Number of replication lag reports retried because the expected report was not received before the next report interval.",
 		unit: "{report}"
 	});
 	#pgVersion;
 	#expectingLagReport = null;
+	#lastAcceptedSendTimeMs = 0;
 	#timer;
-	constructor(lc, shard, db, lagIntervalMs) {
+	constructor(lc, shard, db, pgVersion, reportIntervalMs, reportIntervalJitterMs = 2e3) {
 		this.#lc = lc;
 		this.messagePrefix = `${shard.appID}/${shard.shardNum}${LagReporter.MESSAGE_SUFFIX}`;
 		this.#db = db;
-		this.#lagIntervalMs = lagIntervalMs;
-	}
-	async #getPgVersion() {
-		if (this.#pgVersion === void 0) {
-			const [{ pgVersion }] = await this.#db`
-        SELECT current_setting('server_version_num')::int as "pgVersion"`;
-			this.#pgVersion = pgVersion;
-		}
-		return this.#pgVersion;
+		this.#pgVersion = pgVersion;
+		this.#reportIntervalMs = reportIntervalMs;
+		this.#reportIntervalJitterMs = reportIntervalJitterMs;
 	}
 	get pgVersion() {
 		return this.#pgVersion;
 	}
 	async initiateLagReport(log = false) {
-		const pgVersion = this.#pgVersion ?? await this.#getPgVersion();
 		const now = Date.now();
 		const id = nanoid();
 		const lagReport = {
@@ -412,7 +354,7 @@ var LagReporter = class LagReporter {
 		let commitTimeMs;
 		let lsn;
 		try {
-			if (pgVersion >= 17e4) [{commitTimeMs, lsn}] = await this.#db`
+			if (this.#pgVersion >= 17e4) [{commitTimeMs, lsn}] = await this.#db`
           WITH CTE AS (SELECT extract(epoch from now()) * 1000 AS "commitTimeMs")
           SELECT "commitTimeMs", pg_logical_emit_message(
             false,
@@ -425,7 +367,8 @@ var LagReporter = class LagReporter {
             true
           ) as lsn FROM CTE;
       `;
-			else [{commitTimeMs, lsn}] = await this.#db`
+			else {
+				[{commitTimeMs, lsn}] = await this.#db`
           WITH CTE AS (SELECT extract(epoch from now()) * 1000 as "commitTimeMs")
           SELECT "commitTimeMs", pg_logical_emit_message(
             false,
@@ -437,6 +380,9 @@ var LagReporter = class LagReporter {
             )::text
           ) as lsn FROM CTE;
       `;
+				[{lsn}] = await this.#db`
+          SELECT pg_current_wal_insert_lsn() AS lsn`;
+			}
 		} catch (e) {
 			if (this.#expectingLagReport?.id === id) this.#expectingLagReport = null;
 			throw e;
@@ -472,9 +418,13 @@ var LagReporter = class LagReporter {
 	* of a replication session.
 	*
 	* To account for this, the last emitted lag report is considered "received"
-	* if the stream has advanced beyond the LSN of the report.
+	* if the stream has advanced beyond the WAL insert position read after the
+	* report was emitted (see initiateLagReport()).
+	*
+	* This is unnecessary in PG 17+, where reports are emitted with `flush`.
 	*/
 	checkCurrentLSN(lsn) {
+		if (this.#pgVersion >= 17e4) return;
 		if (this.#expectingLagReport?.lsn && lsn > this.#expectingLagReport.lsn) {
 			this.#lc.info?.(`LSN ${fromBigInt(lsn)} is passed expected lag report ${fromBigInt(this.#expectingLagReport.lsn)}. Processing it as received.`);
 			return this.#processLagReport(this.#expectingLagReport, majorVersionToString(lsn));
@@ -483,8 +433,12 @@ var LagReporter = class LagReporter {
 	stop() {
 		clearTimeout(this.#timer);
 		this.#timer = void 0;
+		this.#expectingLagReport = null;
+		this.#lastAcceptedSendTimeMs = 0;
 	}
-	#scheduleNextReport(delayMs) {
+	#scheduleNextReportWithJitter(delayMs) {
+		const jitter = Math.floor(Math.random() * this.#reportIntervalJitterMs) - this.#reportIntervalJitterMs / 2;
+		delayMs = Math.max(0, delayMs + jitter);
 		this.#expectingLagReport = null;
 		clearTimeout(this.#timer);
 		this.#timer = setTimeout(async () => {
@@ -493,7 +447,7 @@ var LagReporter = class LagReporter {
 				await this.initiateLagReport();
 			} catch (e) {
 				this.#lc.warn?.(`error initiating lag report`, e);
-				this.#scheduleNextReport(this.#lagIntervalMs);
+				this.#scheduleNextReportWithJitter(this.#reportIntervalMs);
 			}
 		}, delayMs);
 	}
@@ -513,26 +467,30 @@ var LagReporter = class LagReporter {
 				await this.initiateLagReport();
 			} catch (e) {
 				this.#lc.warn?.(`error retrying lag report`, e);
-				this.#scheduleNextReport(this.#lagIntervalMs);
+				this.#scheduleNextReportWithJitter(this.#reportIntervalMs);
 			}
-		}, this.#lagIntervalMs);
+		}, this.#reportIntervalMs);
 	}
-	processLagReportMessage(msg) {
+	/**
+	* @param lsn The LSN of the message in the replication stream (i.e. from the
+	*     XLogData header), rather than the `messageLsn` of the message itself,
+	*     which is not guaranteed to be comparable to other stream LSNs (e.g. on
+	*     Aurora). See {@link initiateLagReport}.
+	*/
+	processLagReportMessage(msg, lsn) {
 		assert(msg.prefix === this.messagePrefix, `unexpected message prefix: ${msg.prefix}`);
 		const report = parseLogicalMessageContent(this.#lc, msg, lagReportSchema);
-		return this.#processLagReport(report, toStateVersionString(msg.messageLsn ?? "0/0"));
+		return this.#processLagReport(report, majorVersionToString(lsn));
 	}
 	#processLagReport(report, watermark) {
 		const now = Date.now();
 		const expectedReport = this.#expectingLagReport;
 		let nextSendTimeMs;
-		if (report.id === expectedReport?.id) {
-			nextSendTimeMs = Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
-			this.#scheduleNextReport(nextSendTimeMs - now);
-		} else {
-			this.#lc.debug?.(`received extraneous lag report`, { report });
-			nextSendTimeMs = expectedReport?.sendTimeMs ?? Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
-		}
+		if (expectedReport ? Math.abs(report.sendTimeMs - expectedReport.sendTimeMs) < this.#reportIntervalMs : this.#timer !== void 0 && report.sendTimeMs > this.#lastAcceptedSendTimeMs) {
+			this.#lastAcceptedSendTimeMs = report.sendTimeMs;
+			nextSendTimeMs = Math.max(now, report.sendTimeMs + this.#reportIntervalMs);
+			this.#scheduleNextReportWithJitter(nextSendTimeMs - now);
+		} else nextSendTimeMs = expectedReport?.sendTimeMs ?? Math.max(now, report.sendTimeMs + this.#reportIntervalMs);
 		const { sendTimeMs, commitTimeMs } = report;
 		return [
 			"status",
@@ -553,17 +511,19 @@ var LagReporter = class LagReporter {
 };
 var SET_REPLICA_IDENTITY_DELAY_MS = 50;
 var ChangeMaker = class {
+	#shard;
 	#shardPrefix;
 	#shardConfig;
-	#initialSchema;
+	#replica;
 	#db;
 	#replicaIdentityTimer;
 	#error;
 	#skippedIndexWarnings = /* @__PURE__ */ new Set();
-	constructor({ appID, shardNum }, shardConfig, db, initialSchema) {
-		this.#shardPrefix = `${appID}/${shardNum}`;
+	constructor(shard, shardConfig, db, replica) {
+		this.#shard = shard;
+		this.#shardPrefix = `${shard.appID}/${shard.shardNum}`;
 		this.#shardConfig = shardConfig;
-		this.#initialSchema = initialSchema;
+		this.#replica = replica;
 		this.#db = db;
 	}
 	async makeChanges(lc, lsn, msg) {
@@ -581,6 +541,7 @@ var ChangeMaker = class {
 				lastLogTime: 0
 			};
 			this.#logError(lc, this.#error);
+			await this.#invalidateReplica(lc, err);
 			const message = `Unable to continue replication from LSN ${fromBigInt(lsn)}`;
 			const errorDetails = { error: message };
 			if (err instanceof UnsupportedSchemaChangeError) {
@@ -594,6 +555,18 @@ var ChangeMaker = class {
 			}]];
 		}
 	}
+	#replicaInvalidated = false;
+	async #invalidateReplica(lc, err) {
+		if (this.#replicaInvalidated) return;
+		const sql = this.#db;
+		const { id: replicaID } = this.#replica;
+		const replicasTable = `${upstreamSchema(this.#shard)}.replicas`;
+		lc.warn?.(`deleting invalid replica: ${this.#replica.id}`, err);
+		await sql`
+      DELETE FROM ${sql(replicasTable)} WHERE "id" = ${replicaID}
+    `;
+		this.#replicaInvalidated = true;
+	}
 	#logError(lc, error) {
 		const { lsn, msg, err, lastLogTime } = error;
 		const now = Date.now();
@@ -903,9 +876,9 @@ var ChangeMaker = class {
 		const { publications, ddlDetection } = this.#shardConfig;
 		if (ddlDetection) return [];
 		const currentSchema = await getPublicationInfo(this.#db, publications);
-		const difference = getSchemaDifference(this.#initialSchema, currentSchema);
+		const difference = getSchemaDifference(this.#replica.initialSchema, currentSchema);
 		if (difference !== null) throw new MissingEventTriggerSupport(difference);
-		const orel = this.#initialSchema.tables.find((t) => t.oid === rel.relationOid);
+		const orel = this.#replica.initialSchema.tables.find((t) => t.oid === rel.relationOid);
 		if (!orel) throw new MissingEventTriggerSupport(`relation not in initialSchema: ${stringify(rel)}`);
 		if (relationDifferent(orel, rel)) throw new MissingEventTriggerSupport(`relation has changed within the transaction: ${stringify(orel)} vs ${stringify(rel)}`);
 		return [];
@@ -1071,6 +1044,6 @@ function parseLogicalMessageContent(lc, msg, schema) {
 	}
 }
 //#endregion
-export { initializePostgresChangeSource };
+export { PostgresChangeSource };
 
 //# sourceMappingURL=change-source.js.map