@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.40

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts

@@ -3,35 +3,31 @@ import * as v from '../../../../../shared/src/valita.ts';
 import type { PublishedIndexSpec, PublishedTableSpec } from '../../../db/specs.ts';
 import { type LexiVersion } from '../../../types/lexi-version.ts';
 import { type PostgresDB } from '../../../types/pg.ts';
-import { type ShardConfig, type ShardID } from '../../../types/shards.ts';
+import { type ShardID } from '../../../types/shards.ts';
 import type { Sink } from '../../../types/streams.ts';
 import type { ChangeSource, ChangeStream } from '../change-source.ts';
 import { type Listener } from '../common/change-stream-multiplexer.ts';
-import { type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
 import type { BackfillRequest, DownstreamStatusMessage } from '../protocol/current.ts';
 import type { ChangeStreamMessage } from '../protocol/current/downstream.ts';
 import { streamBackfill } from './backfill-stream.ts';
-import { type InitialSyncOptions, type ReplicaOptions, type ServerContext } from './initial-sync.ts';
+import { type ServerContext } from './initial-sync.ts';
 import type { MessageMessage, MessageRelation as PostgresRelation } from './logical-replication/pgoutput.types.ts';
 import { subscribe } from './logical-replication/stream.ts';
 import { type PublishedTableWithReplicaIdentity } from './schema/published.ts';
 import { type BackupOptions, type InternalShardConfig, type Replica } from './schema/shard.ts';
-interface PurgeLock {
-    release(): Promise<void>;
-}
-/**
- * Initializes a Postgres change source, including the initial sync of the
- * replica, before streaming changes from the corresponding logical replication
- * stream.
- */
-export declare function initializePostgresChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, syncOptions: InitialSyncOptions, context: ServerContext, lagReportIntervalMs?: number, restoreOptions?: RestoreOptions, { backupV5 }?: ReplicaOptions, purgeLock?: PurgeLock | null, streamInboundTimeoutMs?: number | undefined): Promise<InitializeResult>;
 /**
  * Postgres implementation of a {@link ChangeSource} backed by a logical
  * replication stream.
  */
 export declare class PostgresChangeSource implements ChangeSource {
     #private;
-    constructor(lc: LogContext, upstreamUri: string, shard: ShardID, replica: Replica, backupOptions: BackupOptions, context: ServerContext, lagReportIntervalMs: number, textCopy?: boolean | undefined, streamInboundTimeoutMs?: number | undefined, deps?: {
+    constructor(lc: LogContext, upstreamUri: string, shard: ShardID, replica: Replica, pgVersion: number, backupOptions: BackupOptions, context: ServerContext, lagReportIntervalMs: number, textCopy?: boolean | undefined, streamInboundTimeoutMs?: number | undefined, 
+    /**
+     * The watermark below which a stream cannot be started without skipping
+     * transactions, as Postgres would silently start from the slot's position
+     * instead (see `minStartWatermark()` in change-source-init.ts).
+     */
+    minStartWatermark?: string | undefined, deps?: {
         subscribe?: typeof subscribe;
         streamBackfill?: typeof streamBackfill;
     });
@@ -45,7 +41,15 @@ export declare class PostgresChangeSource implements ChangeSource {
 }
 export declare class Acker implements Listener {
     #private;
-    constructor(acks: Sink<bigint>);
+    /**
+     * @param resumeWatermark the watermark the stream resumes after. What came
+     *     before it was received on an earlier connection, and only the
+     *     change-streamer knows whether it has persisted it, so keepalives are
+     *     not acked until the change-streamer has acked it. A change-streamer
+     *     whose SQLite change log is ahead of its backup would otherwise have
+     *     the slot moved past transactions that only that log holds.
+     */
+    constructor(acks: Sink<bigint>, resumeWatermark: string | null);
     onChange(change: ChangeStreamMessage): void;
     ack(watermark: LexiVersion): void;
 }
@@ -59,8 +63,8 @@ export declare class LagReporter {
     #private;
     static readonly MESSAGE_SUFFIX = "/lag-report/v1";
     readonly messagePrefix: string;
-    constructor(lc: LogContext, shard: ShardID, db: PostgresDB, lagIntervalMs: number);
-    get pgVersion(): number | undefined;
+    constructor(lc: LogContext, shard: ShardID, db: PostgresDB, pgVersion: number, reportIntervalMs: number, reportIntervalJitterMs?: number);
+    get pgVersion(): number;
     initiateLagReport(log?: boolean): Promise<{
         firstCommitTimeMs: number;
         nextSendTimeMs: number;
@@ -82,11 +86,20 @@ export declare class LagReporter {
      * of a replication session.
      *
      * To account for this, the last emitted lag report is considered "received"
-     * if the stream has advanced beyond the LSN of the report.
+     * if the stream has advanced beyond the WAL insert position read after the
+     * report was emitted (see initiateLagReport()).
+     *
+     * This is unnecessary in PG 17+, where reports are emitted with `flush`.
      */
     checkCurrentLSN(lsn: bigint): DownstreamStatusMessage | undefined;
     stop(): void;
-    processLagReportMessage(msg: MessageMessage): DownstreamStatusMessage;
+    /**
+     * @param lsn The LSN of the message in the replication stream (i.e. from the
+     *     XLogData header), rather than the `messageLsn` of the message itself,
+     *     which is not guaranteed to be comparable to other stream LSNs (e.g. on
+     *     Aurora). See {@link initiateLagReport}.
+     */
+    processLagReportMessage(msg: MessageMessage, lsn: bigint): DownstreamStatusMessage;
 }
 export declare function relationDifferent(a: PublishedTableSpec, b: PostgresRelation): boolean;
 /**

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAOzD,OAAO,KAAK,EAEV,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,OAAO,EACb,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAMpD,OAAO,KAAK,EAAC,YAAY,EAAE,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,KAAK,EACV,eAAe,EACf,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAC,SAAS,EAAqB,MAAM,iCAAiC,CAAC;AAM9E,OAAO,EAGL,KAAK,iCAAiC,EAEvC,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAQL,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,OAAO,EAEb,MAAM,mBAAmB,CAAC;AAK3B,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EAAC,QAAQ,EAAC,GAAE,cAAiC,EAC7C,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,EAC5B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CAuG3B;AA0HD;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,YAAY;;IAevD,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,aAAa,EACtB,mBAAmB,EAAE,MAAM,EAC3B,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,EAC9B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,EAG3C,IAAI,GAAE;QACJ,SAAS,CAAC,EAAE,OAAO,SAAS,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,cAAc,CAAC;KACnC,EAgCP;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAK1B;IAEK,gBAAgB;;;cAyBrB;IAEK,WAAW,CACf,eAAe,EAAE,MAAM,EACvB,gBAAgB,GAAE,eAAe,EAAO,GACvC,OAAO,CAAC,YAAY,CAAC,CAWvB;IAGK,mBAAmB,CACvB,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,WAAW,EAAE,mBAAmB,EAChC,gBAAgB,EAAE,eAAe,EAAE,GAClC,OAAO,CAAC,YAAY,CAAC,CAgKvB;CAuGF;AAGD,qBAAa,KAAM,YAAW,QAAQ;;IAIpC,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAE7B;IAED,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CA0B1C;IAMD,GAAG,CAAC,SAAS,EAAE,WAAW,QAQzB;CAYF;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAoB/B,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EAMtB;IAWD,IAAI,SAAS,uBAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ;;;OAwElC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS,CAYhE;IAED,IAAI,SAGH;IAwCD,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB,CAUpE;CAuCF;AA2pBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E;AAsBD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,kBAAkB,EACxB,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,EAC1D,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,GACzD,OAAO,CAwET"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAEV,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAA4B,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AAChF,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAKtE,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,KAAK,EAAC,YAAY,EAAE,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EACV,eAAe,EACf,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAC,KAAK,aAAa,EAAC,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAC,SAAS,EAAqB,MAAM,iCAAiC,CAAC;AAK9E,OAAO,EAIL,KAAK,iCAAiC,EACvC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAKL,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,OAAO,EACb,MAAM,mBAAmB,CAAC;AAU3B;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,YAAY;;IAiBvD,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,OAAO,EAChB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,aAAa,EACtB,mBAAmB,EAAE,MAAM,EAC3B,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,EAC9B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS;IAC3C;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,EAGtC,IAAI,GAAE;QACJ,SAAS,CAAC,EAAE,OAAO,SAAS,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,cAAc,CAAC;KACnC,EAyCP;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAI1B;IAEK,gBAAgB;;;cAyBrB;IAEK,WAAW,CACf,eAAe,EAAE,MAAM,EACvB,gBAAgB,GAAE,eAAe,EAAO,GACvC,OAAO,CAAC,YAAY,CAAC,CAsBvB;IAGK,mBAAmB,CACvB,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,WAAW,EAAE,mBAAmB,EAChC,gBAAgB,EAAE,eAAe,EAAE,GAClC,OAAO,CAAC,YAAY,CAAC,CA+KvB;CA+HF;AAGD,qBAAa,KAAM,YAAW,QAAQ;;IAIpC;;;;;;;OAOG;IACH,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI,EAG7D;IAED,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAyB1C;IAMD,GAAG,CAAC,SAAS,EAAE,WAAW,QAQzB;CAYF;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAwB/B,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,MAAM,EACjB,gBAAgB,EAAE,MAAM,EACxB,sBAAsB,GAAE,MAAa,EAQtC;IAED,IAAI,SAAS,WAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ;;;OA+ElC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS,CAehE;IAED,IAAI,SAKH;IA6CD;;;;;OAKG;IACH,uBAAuB,CACrB,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,MAAM,GACV,uBAAuB,CAOzB;CAwDF;AA2rBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E;AAsBD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,kBAAkB,EACxB,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,EAC1D,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,GACzD,OAAO,CAwET"}

package/out/zero-cache/src/services/change-source/pg/change-source.js

@@ -7,113 +7,30 @@ import { mapValues } from "../../../../../shared/src/objects.js";
 import { equals, intersection, symmetricDifferences } from "../../../../../shared/src/set-utils.js";
 import { stringify } from "../../../../../shared/src/bigint-json.js";
 import { upstreamSchema } from "../../../types/shards.js";
-import { Database } from "../../../../../zqlite/src/db.js";
 import { UnsupportedColumnDefaultError, mapPostgresToLiteColumn } from "../../../db/pg-to-lite.js";
 import { getOrCreateCounter } from "../../../observability/metrics.js";
-import { getSubscriptionStateAndContext } from "../../replicator/schema/replication-state.js";
-import { StatementRunner } from "../../../db/statements.js";
-import { connectPgClient, isPostgresError, pgClient } from "../../../types/pg.js";
+import { isPostgresError, pgClient } from "../../../types/pg.js";
+import { UnrecoverableError } from "../../running-state.js";
 import { majorVersionFromString, majorVersionToString } from "../../../types/state-version.js";
 import { fromBigInt, toBigInt, toStateVersionString } from "./lsn.js";
 import { getPublicationInfo, warnForSkippedIndexes } from "./schema/published.js";
 import { replicationEventSchema } from "./schema/ddl.js";
+import "./schema/replica-stage-enum.js";
 import { validate } from "./schema/validation.js";
-import { dropShard, getActiveReplicas, getInternalShardConfig, getReplicaAtVersion, internalPublicationPrefix, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotPrefix } from "./schema/shard.js";
-import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
+import { getInternalShardConfig, getReplicaAtVersion, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotPrefix } from "./schema/shard.js";
 import { mapIndexPredicateColumns } from "../../../db/index-predicate.js";
-import { restoreReplica } from "../common/replica-restore.js";
-import { initReplica } from "../common/replica-schema.js";
 import "../../../types/pg-versions.js";
 import { BackfillManager } from "../common/backfill-manager.js";
 import { ChangeStreamMultiplexer } from "../common/change-stream-multiplexer.js";
 import { subscribe } from "./logical-replication/stream.js";
-import { dropOldReplicasAndSlots } from "./replication-slots.js";
-import { ensureShardSchema } from "./schema/init.js";
-import { initialSync } from "./initial-sync.js";
+import "./initial-sync.js";
 import { streamBackfill } from "./backfill-stream.js";
+import { ReplicationSlotCleanupMonitor } from "./replication-slot-cleanup-monitor.js";
 import { registerReplicationSlotHealthMetrics } from "./replication-slot-health.js";
 import { nanoid } from "nanoid";
 import postgres from "postgres";
 import { PG_ADMIN_SHUTDOWN, PG_INSUFFICIENT_PRIVILEGE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/change-source.ts
-var REPLICA_SLOT_CLEANUP_INTERVAL_MS = 3e4;
-/**
-* Initializes a Postgres change source, including the initial sync of the
-* replica, before streaming changes from the corresponding logical replication
-* stream.
-*/
-async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbFile, syncOptions, context, lagReportIntervalMs = 0, restoreOptions = {}, { backupV5 } = { backupV5: true }, purgeLock, streamInboundTimeoutMs) {
-	const db = await connectPgClient(lc, upstreamURI, "change-source-init");
-	try {
-		await ensureShardSchema(lc, db, shard, syncOptions.installPartialIndexTriggers);
-		const restoredReplica = await selectAndRestoreReplica(lc, db, shard, replicaDbFile, restoreOptions);
-		let initialSyncedReplica;
-		await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, async (log, tx) => {
-			purgeLock?.release();
-			initialSyncedReplica = await initialSync(log, shard, tx, upstreamURI, syncOptions, context, { backupV5 });
-		});
-		const replica = new Database(lc, replicaDbFile);
-		const subscriptionState = getSubscriptionStateAndContext(new StatementRunner(replica));
-		replica.close();
-		const upstreamReplica = await checkAndUpdateUpstream(lc, db, shard, subscriptionState, (initialSyncedReplica ?? restoredReplica)?.id);
-		const backupPath = initialSyncedReplica ? initialSyncedReplica.backupPath : backupV5 ? String(Date.now()) : restoredReplica?.backupPath ?? null;
-		return {
-			subscriptionState,
-			changeSource: new PostgresChangeSource(lc, upstreamURI, shard, upstreamReplica, {
-				backupPath,
-				backupV5
-			}, context, lagReportIntervalMs, syncOptions.textCopy, streamInboundTimeoutMs),
-			destinationBackupURL: backupPath && restoreOptions.litestream?.backupURL ? new URL(backupPath, restoreOptions.litestream.backupURL).toString() : restoreOptions.litestream?.backupURL,
-			replicaID: upstreamReplica.id,
-			waitForBackupBeforeServing: initialSyncedReplica !== void 0 || backupPath !== (restoredReplica?.backupPath ?? null)
-		};
-	} finally {
-		await db.end();
-	}
-}
-async function selectAndRestoreReplica(lc, sql, shard, replicaFile, { litestream, constraints }) {
-	const replicas = (await getActiveReplicas(lc, sql, shard)).filter(({ generation }) => generation === (constraints?.replicaVersion ?? generation));
-	if (replicas.length === 0) {
-		lc.info?.(`no suitable replicas to restore from`, { replicas });
-		return;
-	}
-	const [replica] = replicas;
-	if (litestream?.backupURL) {
-		const { backupURL: backupBaseURL } = litestream;
-		const { slot, backupPath, confirmedFlushLsn } = replica;
-		const backupURL = new URL(backupPath ?? "", backupBaseURL).toString();
-		lc.info?.(`restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`, { replicas });
-		await restoreReplica(lc, {
-			...litestream,
-			backupURL
-		}, replicaFile, constraints);
-	}
-	return replica;
-}
-async function checkAndUpdateUpstream(lc, sql, shard, { replicaVersion, publications: subscribed, initialSyncContext }, replicaID) {
-	const upstreamReplica = await getReplicaAtVersion(lc, sql, shard, replicaVersion, replicaID, initialSyncContext);
-	if (!upstreamReplica) throw new AutoResetSignal(`No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`);
-	const requested = shard.publications.toSorted();
-	const replicated = upstreamReplica.publications.filter((p) => !p.startsWith(internalPublicationPrefix(shard))).sort();
-	if (!deepEqual(requested, replicated)) {
-		lc.warn?.(`Dropping shard to change publications to: [${requested}]`);
-		await sql.unsafe(dropShard(shard.appID, shard.shardNum));
-		throw new AutoResetSignal(`Requested publications [${requested}] do not match configured publications: [${replicated}]`);
-	}
-	if (!deepEqual(upstreamReplica.publications, subscribed)) throw new AutoResetSignal(`Upstream publications [${upstreamReplica.publications}] do not match subscribed publications [${subscribed}]`);
-	const exists = await sql`
-    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};
-  `.values();
-	if (exists.length !== subscribed.length) throw new AutoResetSignal(`Upstream publications [${exists.flat()}] do not contain all subscribed publications [${subscribed}]`);
-	const { slot } = upstreamReplica;
-	const result = await sql`
-    SELECT restart_lsn as "restartLSN", wal_status as "walStatus" FROM pg_replication_slots
-      WHERE slot_name = ${slot}`;
-	if (result.length === 0) throw new AutoResetSignal(`replication slot ${slot} is missing`);
-	const [{ restartLSN, walStatus }] = result;
-	if (restartLSN === null || walStatus === "lost") throw new AutoResetSignal(`replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`);
-	return upstreamReplica;
-}
 var MAX_LOW_PRIORITY_DELAY_MS = 1e3;
 /**
 * Postgres implementation of a {@link ChangeSource} backed by a logical
@@ -125,36 +42,39 @@ var PostgresChangeSource = class {
 	#upstreamUri;
 	#shard;
 	#replica;
+	#slotCleanupMonitor;
 	#backupOptions;
 	#context;
 	#lagReporter;
 	#textCopy;
 	#streamInboundTimeoutMs;
+	#minStartWatermark;
 	#subscribe;
 	#streamBackfill;
 	#stopped = false;
-	constructor(lc, upstreamUri, shard, replica, backupOptions, context, lagReportIntervalMs, textCopy, streamInboundTimeoutMs, deps = {}) {
+	constructor(lc, upstreamUri, shard, replica, pgVersion, backupOptions, context, lagReportIntervalMs, textCopy, streamInboundTimeoutMs, minStartWatermark, deps = {}) {
 		this.#lc = lc.withContext("component", "change-source");
 		this.#subscribe = deps.subscribe ?? subscribe;
 		this.#streamBackfill = deps.streamBackfill ?? streamBackfill;
 		this.#db = pgClient(lc, upstreamUri, "replication-monitor", {
-			max: 1,
-			["idle_timeout"]: 60
+			max: 3,
+			idle_timeout: 60
 		});
 		this.#upstreamUri = upstreamUri;
 		this.#shard = shard;
 		this.#replica = replica;
+		this.#slotCleanupMonitor = new ReplicationSlotCleanupMonitor(lc.withContext("component", "replication-slot-monitor"), shard, this.#db, pgVersion, replica.slot);
 		this.#backupOptions = backupOptions;
 		this.#context = context;
 		this.#textCopy = textCopy ?? false;
 		this.#streamInboundTimeoutMs = streamInboundTimeoutMs;
-		this.#lagReporter = lagReportIntervalMs > 0 ? new LagReporter(lc.withContext("component", "lag-reporter"), shard, this.#db, lagReportIntervalMs) : null;
+		this.#minStartWatermark = minStartWatermark;
+		this.#lagReporter = lagReportIntervalMs > 0 ? new LagReporter(lc.withContext("component", "lag-reporter"), shard, this.#db, pgVersion, lagReportIntervalMs) : null;
 		registerReplicationSlotHealthMetrics(this.#lc, this.#db, replica.slot, () => this.#stopped);
 	}
 	async stop() {
 		this.#stopped = true;
 		this.#lagReporter?.stop();
-		clearTimeout(this.#cleanupTimer);
 		await this.#db.end();
 	}
 	async startLagReporter() {
@@ -173,24 +93,32 @@ var PostgresChangeSource = class {
 		const config = await getInternalShardConfig(this.#db, this.#shard);
 		const { slot } = this.#replica;
 		this.#lc.info?.(`starting replication stream@${slot}`);
-		return this.startStreamInternal(slot, clientWatermark, config, backfillRequests);
+		await this.#checkConfirmedFlushLSN(slot, clientWatermark);
+		const changeStream = await this.startStreamInternal(slot, clientWatermark, config, backfillRequests);
+		const { signal } = changeStream.changes;
+		if (!signal.aborted) {
+			this.#slotCleanupMonitor.start();
+			signal.addEventListener("abort", () => this.#slotCleanupMonitor.stop());
+		}
+		return changeStream;
 	}
 	async startStreamInternal(slot, clientWatermark, shardConfig, backfillRequests) {
+		if (this.#minStartWatermark !== void 0 && majorVersionFromString(clientWatermark) < majorVersionFromString(this.#minStartWatermark)) throw new UnrecoverableError(`cannot start replication stream@${slot} from ${clientWatermark}, which is before its minimum start watermark ${this.#minStartWatermark}: transactions in between would be silently skipped`);
 		const clientStart = majorVersionFromString(clientWatermark) + 1n;
 		const { messages, acks } = await this.#subscribe(this.#lc, this.#db, slot, [...shardConfig.publications], clientStart, void 0, void 0, this.#streamInboundTimeoutMs);
-		const acker = new Acker(acks);
+		const acker = new Acker(acks, clientWatermark);
 		const changes = new ChangeStreamMultiplexer(this.#lc, clientWatermark);
 		const backfillManager = new BackfillManager(this.#lc, changes, (req) => this.#streamBackfill(this.#lc, this.#upstreamUri, this.#replica, req, { textCopy: this.#textCopy }));
 		changes.addProducers(messages, backfillManager).addListeners(backfillManager, acker);
 		backfillManager.run(clientWatermark, backfillRequests);
-		const changeMaker = new ChangeMaker(this.#shard, shardConfig, this.#db, this.#replica.initialSchema);
+		const changeMaker = new ChangeMaker(this.#shard, shardConfig, this.#db, this.#replica);
 		/**
 		* Determines if the incoming message is transactional, otherwise handling
 		* non-transactional messages with a downstream status message.
 		*/
 		const isTransactionalMessage = (lsn, msg) => {
 			if (msg.tag === "message" && msg.prefix === this.#lagReporter?.messagePrefix) {
-				changes.pushStatus(this.#lagReporter.processLagReportMessage(msg));
+				changes.pushStatus(this.#lagReporter.processLagReportMessage(msg, lsn));
 				return false;
 			}
 			const status = this.#lagReporter?.checkCurrentLSN(lsn);
@@ -256,6 +184,31 @@ var PostgresChangeSource = class {
 			acks: { push: (status) => acker.ack(status[2].watermark) }
 		};
 	}
+	/**
+	* Warns if the stream is about to start from before the slot's
+	* `confirmed_flush_lsn`, from which Postgres starts streaming instead.
+	*
+	* This is benign if the slot was only acked past `clientWatermark` for
+	* keepalives (i.e. with no transactions in between), which the
+	* {@link Acker} does on a caught-up stream. It is therefore not an error
+	* (`minStartWatermark` is the bound that is), but it is the evidence needed
+	* to diagnose a skipped interval.
+	*/
+	async #checkConfirmedFlushLSN(slot, clientWatermark) {
+		try {
+			const [row] = await this.#db`
+        SELECT confirmed_flush_lsn AS "confirmedFlushLSN"
+          FROM pg_replication_slots WHERE slot_name = ${slot}`;
+			const confirmed = row?.confirmedFlushLSN;
+			if (!confirmed) return;
+			const start = majorVersionFromString(clientWatermark);
+			const confirmedLSN = toBigInt(confirmed);
+			if (start < confirmedLSN) this.#lc.warn?.(`starting replication stream@${slot} from ${clientWatermark}, which is ${confirmedLSN - start} bytes before the slot's confirmed_flush_lsn ${confirmed} (${toStateVersionString(confirmed)}). Postgres starts from the latter, which skips nothing only if the slot was acked past ${clientWatermark} for keepalives.`);
+			else this.#lc.info?.(`replication slot ${slot} confirmed_flush_lsn: ${confirmed} (${toStateVersionString(confirmed)})`);
+		} catch (e) {
+			this.#lc.warn?.(`unable to read the position of slot ${slot}`, e);
+		}
+	}
 	async #logCurrentReplicaInfo() {
 		try {
 			const replica = await getReplicaAtVersion(this.#lc, this.#db, this.#shard, this.#replica.generation, this.#replica.id);
@@ -284,6 +237,15 @@ var PostgresChangeSource = class {
 		const sql = this.#db;
 		const { id: replicaID, slot } = this.#replica;
 		const replicasTable = `${upstreamSchema(this.#shard)}.replicas`;
+		await sql`
+      UPDATE ${sql(replicasTable)} 
+        SET "stage" = ${1},
+            "version" = "generation",
+            "subscriberContext" = ${this.#context},
+            "backupPath" = ${this.#backupOptions.backupPath},
+            "backupV5" = ${this.#backupOptions.backupV5},
+            "pgChangeLog" = ${this.#backupOptions.pgChangeLog}
+        WHERE id = ${replicaID}`;
 		const result = await sql`
       SELECT pg_terminate_backend(active_pid) as terminated, active_pid as pid
         FROM pg_replication_slots
@@ -306,35 +268,22 @@ var PostgresChangeSource = class {
 			throw new AbortError(`replication slot ${slot} is missing. A different replication-manager should now be running on a new replication slot.`);
 		}
 		this.#lc.info?.(`terminated replication slots: ${JSON.stringify(result)}`);
-		await sql`
-      UPDATE ${sql(replicasTable)} 
-        SET "subscriberContext" = ${this.#context},
-            "backupPath" = ${this.#backupOptions.backupPath},
-            "backupV5" = ${this.#backupOptions.backupV5}
-        WHERE id = ${replicaID}`;
-		this.#cleanUpOlderReplicasAndSlots();
-	}
-	#cleanupTimer;
-	async #cleanUpOlderReplicasAndSlots() {
-		clearTimeout(this.#cleanupTimer);
-		try {
-			const result = await dropOldReplicasAndSlots(this.#lc, this.#db, this.#shard, this.#replica.rank);
-			if (result.draining === 0) {
-				this.#lc.info?.(`finished cleaning up replicas and slots`, { result });
-				return;
-			}
-			this.#lc.info?.(`old slots still draining`, { result });
-		} catch (e) {
-			this.#lc.warn?.(`error dropping replication slots`, e);
-		}
-		this.#cleanupTimer = setTimeout(() => this.#cleanUpOlderReplicasAndSlots(), REPLICA_SLOT_CLEANUP_INTERVAL_MS);
 	}
 };
 var Acker = class {
 	#acks;
-	#waitingForDownstreamAck = null;
-	constructor(acks) {
+	#waitingForDownstreamAck;
+	/**
+	* @param resumeWatermark the watermark the stream resumes after. What came
+	*     before it was received on an earlier connection, and only the
+	*     change-streamer knows whether it has persisted it, so keepalives are
+	*     not acked until the change-streamer has acked it. A change-streamer
+	*     whose SQLite change log is ahead of its backup would otherwise have
+	*     the slot moved past transactions that only that log holds.
+	*/
+	constructor(acks, resumeWatermark) {
 		this.#acks = acks;
+		this.#waitingForDownstreamAck = resumeWatermark;
 	}
 	onChange(change) {
 		switch (change[0]) {
@@ -373,33 +322,28 @@ var LagReporter = class LagReporter {
 	#lc;
 	messagePrefix;
 	#db;
-	#lagIntervalMs;
+	#reportIntervalMs;
+	#reportIntervalJitterMs;
 	#lagReportRetries = getOrCreateCounter("replication", "lag_report_retries", {
 		description: "Number of replication lag reports retried because the expected report was not received before the next report interval.",
 		unit: "{report}"
 	});
 	#pgVersion;
 	#expectingLagReport = null;
+	#lastAcceptedSendTimeMs = 0;
 	#timer;
-	constructor(lc, shard, db, lagIntervalMs) {
+	constructor(lc, shard, db, pgVersion, reportIntervalMs, reportIntervalJitterMs = 2e3) {
 		this.#lc = lc;
 		this.messagePrefix = `${shard.appID}/${shard.shardNum}${LagReporter.MESSAGE_SUFFIX}`;
 		this.#db = db;
-		this.#lagIntervalMs = lagIntervalMs;
-	}
-	async #getPgVersion() {
-		if (this.#pgVersion === void 0) {
-			const [{ pgVersion }] = await this.#db`
-        SELECT current_setting('server_version_num')::int as "pgVersion"`;
-			this.#pgVersion = pgVersion;
-		}
-		return this.#pgVersion;
+		this.#pgVersion = pgVersion;
+		this.#reportIntervalMs = reportIntervalMs;
+		this.#reportIntervalJitterMs = reportIntervalJitterMs;
 	}
 	get pgVersion() {
 		return this.#pgVersion;
 	}
 	async initiateLagReport(log = false) {
-		const pgVersion = this.#pgVersion ?? await this.#getPgVersion();
 		const now = Date.now();
 		const id = nanoid();
 		const lagReport = {
@@ -412,7 +356,7 @@ var LagReporter = class LagReporter {
 		let commitTimeMs;
 		let lsn;
 		try {
-			if (pgVersion >= 17e4) [{commitTimeMs, lsn}] = await this.#db`
+			if (this.#pgVersion >= 17e4) [{commitTimeMs, lsn}] = await this.#db`
           WITH CTE AS (SELECT extract(epoch from now()) * 1000 AS "commitTimeMs")
           SELECT "commitTimeMs", pg_logical_emit_message(
             false,
@@ -425,7 +369,8 @@ var LagReporter = class LagReporter {
             true
           ) as lsn FROM CTE;
       `;
-			else [{commitTimeMs, lsn}] = await this.#db`
+			else {
+				[{commitTimeMs, lsn}] = await this.#db`
           WITH CTE AS (SELECT extract(epoch from now()) * 1000 as "commitTimeMs")
           SELECT "commitTimeMs", pg_logical_emit_message(
             false,
@@ -437,6 +382,9 @@ var LagReporter = class LagReporter {
             )::text
           ) as lsn FROM CTE;
       `;
+				[{lsn}] = await this.#db`
+          SELECT pg_current_wal_insert_lsn() AS lsn`;
+			}
 		} catch (e) {
 			if (this.#expectingLagReport?.id === id) this.#expectingLagReport = null;
 			throw e;
@@ -472,9 +420,13 @@ var LagReporter = class LagReporter {
 	* of a replication session.
 	*
 	* To account for this, the last emitted lag report is considered "received"
-	* if the stream has advanced beyond the LSN of the report.
+	* if the stream has advanced beyond the WAL insert position read after the
+	* report was emitted (see initiateLagReport()).
+	*
+	* This is unnecessary in PG 17+, where reports are emitted with `flush`.
 	*/
 	checkCurrentLSN(lsn) {
+		if (this.#pgVersion >= 17e4) return;
 		if (this.#expectingLagReport?.lsn && lsn > this.#expectingLagReport.lsn) {
 			this.#lc.info?.(`LSN ${fromBigInt(lsn)} is passed expected lag report ${fromBigInt(this.#expectingLagReport.lsn)}. Processing it as received.`);
 			return this.#processLagReport(this.#expectingLagReport, majorVersionToString(lsn));
@@ -483,8 +435,12 @@ var LagReporter = class LagReporter {
 	stop() {
 		clearTimeout(this.#timer);
 		this.#timer = void 0;
+		this.#expectingLagReport = null;
+		this.#lastAcceptedSendTimeMs = 0;
 	}
-	#scheduleNextReport(delayMs) {
+	#scheduleNextReportWithJitter(delayMs) {
+		const jitter = Math.floor(Math.random() * this.#reportIntervalJitterMs) - this.#reportIntervalJitterMs / 2;
+		delayMs = Math.max(0, delayMs + jitter);
 		this.#expectingLagReport = null;
 		clearTimeout(this.#timer);
 		this.#timer = setTimeout(async () => {
@@ -493,7 +449,7 @@ var LagReporter = class LagReporter {
 				await this.initiateLagReport();
 			} catch (e) {
 				this.#lc.warn?.(`error initiating lag report`, e);
-				this.#scheduleNextReport(this.#lagIntervalMs);
+				this.#scheduleNextReportWithJitter(this.#reportIntervalMs);
 			}
 		}, delayMs);
 	}
@@ -513,26 +469,30 @@ var LagReporter = class LagReporter {
 				await this.initiateLagReport();
 			} catch (e) {
 				this.#lc.warn?.(`error retrying lag report`, e);
-				this.#scheduleNextReport(this.#lagIntervalMs);
+				this.#scheduleNextReportWithJitter(this.#reportIntervalMs);
 			}
-		}, this.#lagIntervalMs);
+		}, this.#reportIntervalMs);
 	}
-	processLagReportMessage(msg) {
+	/**
+	* @param lsn The LSN of the message in the replication stream (i.e. from the
+	*     XLogData header), rather than the `messageLsn` of the message itself,
+	*     which is not guaranteed to be comparable to other stream LSNs (e.g. on
+	*     Aurora). See {@link initiateLagReport}.
+	*/
+	processLagReportMessage(msg, lsn) {
 		assert(msg.prefix === this.messagePrefix, `unexpected message prefix: ${msg.prefix}`);
 		const report = parseLogicalMessageContent(this.#lc, msg, lagReportSchema);
-		return this.#processLagReport(report, toStateVersionString(msg.messageLsn ?? "0/0"));
+		return this.#processLagReport(report, majorVersionToString(lsn));
 	}
 	#processLagReport(report, watermark) {
 		const now = Date.now();
 		const expectedReport = this.#expectingLagReport;
 		let nextSendTimeMs;
-		if (report.id === expectedReport?.id) {
-			nextSendTimeMs = Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
-			this.#scheduleNextReport(nextSendTimeMs - now);
-		} else {
-			this.#lc.debug?.(`received extraneous lag report`, { report });
-			nextSendTimeMs = expectedReport?.sendTimeMs ?? Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
-		}
+		if (expectedReport ? Math.abs(report.sendTimeMs - expectedReport.sendTimeMs) < this.#reportIntervalMs : this.#timer !== void 0 && report.sendTimeMs > this.#lastAcceptedSendTimeMs) {
+			this.#lastAcceptedSendTimeMs = report.sendTimeMs;
+			nextSendTimeMs = Math.max(now, report.sendTimeMs + this.#reportIntervalMs);
+			this.#scheduleNextReportWithJitter(nextSendTimeMs - now);
+		} else nextSendTimeMs = expectedReport?.sendTimeMs ?? Math.max(now, report.sendTimeMs + this.#reportIntervalMs);
 		const { sendTimeMs, commitTimeMs } = report;
 		return [
 			"status",
@@ -553,17 +513,19 @@ var LagReporter = class LagReporter {
 };
 var SET_REPLICA_IDENTITY_DELAY_MS = 50;
 var ChangeMaker = class {
+	#shard;
 	#shardPrefix;
 	#shardConfig;
-	#initialSchema;
+	#replica;
 	#db;
 	#replicaIdentityTimer;
 	#error;
 	#skippedIndexWarnings = /* @__PURE__ */ new Set();
-	constructor({ appID, shardNum }, shardConfig, db, initialSchema) {
-		this.#shardPrefix = `${appID}/${shardNum}`;
+	constructor(shard, shardConfig, db, replica) {
+		this.#shard = shard;
+		this.#shardPrefix = `${shard.appID}/${shard.shardNum}`;
 		this.#shardConfig = shardConfig;
-		this.#initialSchema = initialSchema;
+		this.#replica = replica;
 		this.#db = db;
 	}
 	async makeChanges(lc, lsn, msg) {
@@ -581,6 +543,7 @@ var ChangeMaker = class {
 				lastLogTime: 0
 			};
 			this.#logError(lc, this.#error);
+			await this.#invalidateReplica(lc, err);
 			const message = `Unable to continue replication from LSN ${fromBigInt(lsn)}`;
 			const errorDetails = { error: message };
 			if (err instanceof UnsupportedSchemaChangeError) {
@@ -594,6 +557,18 @@ var ChangeMaker = class {
 			}]];
 		}
 	}
+	#replicaInvalidated = false;
+	async #invalidateReplica(lc, err) {
+		if (this.#replicaInvalidated) return;
+		const sql = this.#db;
+		const { id: replicaID } = this.#replica;
+		const replicasTable = `${upstreamSchema(this.#shard)}.replicas`;
+		lc.warn?.(`deleting invalid replica: ${this.#replica.id}`, err);
+		await sql`
+      DELETE FROM ${sql(replicasTable)} WHERE "id" = ${replicaID}
+    `;
+		this.#replicaInvalidated = true;
+	}
 	#logError(lc, error) {
 		const { lsn, msg, err, lastLogTime } = error;
 		const now = Date.now();
@@ -903,9 +878,9 @@ var ChangeMaker = class {
 		const { publications, ddlDetection } = this.#shardConfig;
 		if (ddlDetection) return [];
 		const currentSchema = await getPublicationInfo(this.#db, publications);
-		const difference = getSchemaDifference(this.#initialSchema, currentSchema);
+		const difference = getSchemaDifference(this.#replica.initialSchema, currentSchema);
 		if (difference !== null) throw new MissingEventTriggerSupport(difference);
-		const orel = this.#initialSchema.tables.find((t) => t.oid === rel.relationOid);
+		const orel = this.#replica.initialSchema.tables.find((t) => t.oid === rel.relationOid);
 		if (!orel) throw new MissingEventTriggerSupport(`relation not in initialSchema: ${stringify(rel)}`);
 		if (relationDifferent(orel, rel)) throw new MissingEventTriggerSupport(`relation has changed within the transaction: ${stringify(orel)} vs ${stringify(rel)}`);
 		return [];
@@ -1071,6 +1046,6 @@ function parseLogicalMessageContent(lc, msg, schema) {
 	}
 }
 //#endregion
-export { initializePostgresChangeSource };
+export { PostgresChangeSource };
 
 //# sourceMappingURL=change-source.js.map