@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.40

package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts

@@ -1,9 +1,10 @@
 import type { Readable } from 'node:stream';
 import type { LogContext } from '@rocicorp/logger';
-import type postgres from 'postgres';
+import postgres from 'postgres';
 import { type PostgresDB } from '../../../types/pg.ts';
 import { type ShardID } from '../../../types/shards.ts';
-import { type BackupOptions } from './schema/shard.ts';
+import { type ReplicaStage } from './schema/replica-stage-enum.ts';
+import { type BackupOptions, type ReplicaState } from './schema/shard.ts';
 export type ReplicationSlot = {
     slot_name: string;
     consistent_point: string;
@@ -14,21 +15,54 @@ export type ReplicationSlotResult<T> = {
     slot: ReplicationSlot;
     capturedSnapshot: T;
     initialSession: Readable;
+    replica: ReplicaState;
+};
+export type ReservedSlot = {
+    slot: string;
+    reservation: Readable;
+    replica: ReplicaState;
 };
 export type CreateSlotSpec = {
     slotName: string;
     failover?: boolean;
     temporary?: boolean;
-    lockTimeout?: number;
+    lockTimeout?: number | undefined;
+    diagnosticsDB?: PostgresDB | undefined;
+};
+export declare const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60000;
+/** Thrown when the client-side timeout for creating a slot is reached. */
+export declare class SlotCreationTimeoutError extends Error {
+    readonly name = "SlotCreationTimeoutError";
+}
+export declare function createReplicationSlot(lc: LogContext, session: postgres.Sql, { slotName, failover, temporary, lockTimeout, diagnosticsDB, }: CreateSlotSpec): Promise<ReplicationSlot>;
+/**
+ * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting
+ * on: the walsender's wait event and blocking pids, and the oldest
+ * transactions holding an xid. Slot creation waits for every such transaction
+ * on the instance (in all databases), each with its own lock_timeout, so a
+ * chain of overlapping transactions can exceed the client-side timeout
+ * without the lock_timeout ever firing.
+ *
+ * Query text is deliberately omitted since it can contain row values.
+ *
+ * Exported for testing.
+ */
+export declare function logSlotCreationBlockers(lc: LogContext, sql: PostgresDB, walsenderPID: number, slotName: string): Promise<void>;
+export type CreateReplicaAndSlotOptions = {
+    lockTimeout?: number | undefined;
+    maxBlockedMs?: number | undefined;
+    retryDelayMs?: number | undefined;
 };
-export declare function createReplicationSlot(lc: LogContext, session: postgres.Sql, { slotName, failover, temporary, lockTimeout, }: CreateSlotSpec): Promise<ReplicationSlot>;
 /**
- * Replica and slot creation involves two sessions for proper coordination
- * with other replica management logic:
+ * Replica and slot creation involves several sessions for proper
+ * coordination with other replica management logic:
  *
- * * A normal transaction is started and acquires an advisory lock for
+ * * A dedicated session acquires a (session-level) advisory lock for
  *   replica slot management. This is the same lock that cleanup logic
- *   acquires before cleaning up replication slots.
+ *   acquires before cleaning up replication slots. The lock is held
+ *   outside of a transaction so that the session is not "idle in
+ *   transaction" while slot creation waits on upstream transactions,
+ *   which some upstreams terminate (e.g. idle_in_transaction_session_timeout).
  * * With the lock held, a new replication slot is created in a
  *   replication session. The API of CREATE_REPLICATION_SLOT is such
  *   that it cannot be done in a transaction, and cannot be followed by
@@ -37,6 +71,14 @@ export declare function createReplicationSlot(lc: LogContext, session: postgres.
  * * Once the slot is created, the slot and replica information are recorded
  *   in the `replicas` table before releasing the lock.
  *
+ * Slot creation waits for all transactions on the upstream instance that
+ * hold an xid, so it can be blocked by long-running transactions. A blocked
+ * attempt is abandoned (see {@link CREATE_REPLICATION_SLOT_TIMEOUT_MS}) and
+ * retried in-process, as it leaves no durable state: the `replicas` row is
+ * only inserted after the slot has been created. Retries continue until the
+ * slot is created or until `maxBlockedMs` has elapsed, after which the error
+ * is thrown.
+ *
  * This locking ensures that:
  * 1. multiple replication managers attempting to create a replication slot
  *    will not use the same name for the replication slot (which is selected
@@ -61,22 +103,25 @@ export declare function createReplicationSlot(lc: LogContext, session: postgres.
  * PostgresChangeSource takes over the slot to process the stream in earnest.
  * It is also returned for cleanup in the case of failures (and for control
  * in unit tests).
+ *
+ * For replica's created in the `Restore` stage, the `generation` is
+ * initialized to an empty string instead of the slot's consistent point,
+ * since the slot is not being used for initial sync. It is the
+ * responsibiliy of the caller to call {@link initRestoreReplica} when
+ * the source replica has been selected for restoring, which will carry
+ * over the `generation` and other relevant values from the source.
  */
-export declare function createReplicaAndSlot<T>(lc: LogContext, sql: PostgresDB, sessionName: string, shard: ShardID, replicaID: string, failover: boolean, backupOptions: BackupOptions, captureSnapshot: (snapshot: string) => Promise<T>): Promise<ReplicationSlotResult<T>>;
+export declare function createReplicaAndSlot<T>(lc: LogContext, sql: PostgresDB, sessionName: string, shard: ShardID, epoch: number, replicaID: string, failover: boolean, backupOptions: BackupOptions, captureSnapshot: (snapshot: string) => Promise<T>, stage: ReplicaStage, { lockTimeout, maxBlockedMs, retryDelayMs, }?: CreateReplicaAndSlotOptions): Promise<ReplicationSlotResult<T>>;
 /**
- * Deletes "old" replicas (i.e. those with a lower rank than the current)
- * and attempts to drop replication slots that are not associated with any
- * replica.
+ * Claims the inactive slot of the given replica for use by this task, opening
+ * a replication session to mark it as active. The process should follow up by
+ * taking over the slot via the PostgresChangeSource.
  *
- * If a slot could not be dropped because there is still an active subscriber,
- * it will be reflected in the `draining` count that is returned. When there
- * are draining slots, the method should be retried until all orphaned slots
- * have been dropped.
+ * The replica is identified by its ID (and not just its slot) because rows of
+ * replicas whose slots were dropped outside of zero-cache's control may share
+ * the slot name.
  */
-export declare function dropOldReplicasAndSlots(lc: LogContext, sql: PostgresDB, shard: ShardID, beforeRank: bigint): Promise<{
-    dropped: number;
-    active: number;
-    draining: number;
-}>;
+export declare function claimSlotForResumption(lc: LogContext, sql: PostgresDB, shard: ShardID, _sessionName: string, { id: replicaID, slot }: Pick<ReplicaState, 'id' | 'slot'>): Promise<ReservedSlot | null>;
+export declare function dropInactiveSlotsAndReplicas(lc: LogContext, sql: PostgresDB, shard: ShardID, slots: string[]): Promise<void>;
 export declare function slotPoolSuffix(n: number): string;
 //# sourceMappingURL=replication-slots.d.ts.map

package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAK1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAOtE,OAAO,EAKL,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,qBAAqB,CAAC,CAAC,IAAI;IACrC,IAAI,EAAE,eAAe,CAAC;IACtB,gBAAgB,EAAE,CAAC,CAAC;IACpB,cAAc,EAAE,QAAQ,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IAGjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAInB,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAoBF,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,EACE,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,WAAoC,GACrC,EAAE,cAAc,GAChB,OAAO,CAAC,eAAe,CAAC,CAwC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GAChD,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAuFnC;AAED;;;;;;;;;GASG;AACH,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC,CAY9D;AA+DD,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,UAUvC"}
+{"version":3,"file":"replication-slots.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAM1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,QAA2C,MAAM,UAAU,CAAC;AAMnE,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAUtE,OAAO,EAGL,KAAK,YAAY,EAClB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAOL,KAAK,aAAa,EAClB,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,qBAAqB,CAAC,CAAC,IAAI;IACrC,IAAI,EAAE,eAAe,CAAC;IACtB,gBAAgB,EAAE,CAAC,CAAC;IACpB,cAAc,EAAE,QAAQ,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,QAAQ,CAAC;IACtB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IAGjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAInB,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAIjC,aAAa,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;CACxC,CAAC;AAUF,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAOzD,0EAA0E;AAC1E,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,QAAQ,CAAC,IAAI,8BAA8B;CAC5C;AAKD,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,EACE,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,WAAoC,EACpC,aAAa,GACd,EAAE,cAAc,GAChB,OAAO,CAAC,eAAe,CAAC,CAwD1B;AAKD;;;;;;;;;;;GAWG;AACH,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,iBAmCjB;AA0BD,MAAM,MAAM,2BAA2B,GAAG;IAExC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC,CAAC;AAsBF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,KAAK,EAAE,YAAY,EACnB,EACE,WAAW,EACX,YAAmD,EACnD,YAAmD,GACpD,GAAE,2BAAgC,GAClC,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAsKnC;AAED;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,YAAY,EAAE,MAAM,EACpB,EAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAC,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,GAAG,MAAM,CAAC,GACvD,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CA0D9B;AAED,wBAAgB,4BAA4B,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EAAE,iBA+BhB;AAmFD,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,UAUvC"}

package/out/zero-cache/src/services/change-source/pg/replication-slots.js

@@ -1,35 +1,124 @@
+import { sleep } from "../../../../../shared/src/sleep.js";
+import { must } from "../../../../../shared/src/must.js";
 import { upstreamSchema } from "../../../types/shards.js";
 import { isPostgresError } from "../../../types/pg.js";
+import { orTimeout } from "../../../types/timeout.js";
 import { toBigInt, toStateVersionString } from "./lsn.js";
 import { runTx } from "../../../db/run-transaction.js";
-import { createReplica, metadataPublicationName, replicationSlotExpression, replicationSlotPrefix } from "./schema/shard.js";
-import { orTimeout } from "../../../types/timeout.js";
-import { createReplicationSessionFor, keepSlotActiveUntilTakenOver } from "./logical-replication/stream.js";
-import { PG_CONFIGURATION_LIMIT_EXCEEDED, PG_INSUFFICIENT_PRIVILEGE } from "@drdgvhbh/postgres-error-codes";
+import "./schema/replica-stage-enum.js";
+import { createReplica, getReplicaState, getRestoreCandidates, metadataPublicationName, replicationSlotExpression, replicationSlotPrefix } from "./schema/shard.js";
+import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
+import "../../../types/pg-versions.js";
+import { createReplicationSessionFor, endSession, extractConnectionConfig, keepSlotActiveUntilTakenOver } from "./logical-replication/stream.js";
+import { defu } from "defu";
+import postgres from "postgres";
+import { PG_CONFIGURATION_LIMIT_EXCEEDED, PG_INSUFFICIENT_PRIVILEGE, PG_LOCK_NOT_AVAILABLE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/replication-slots.ts
-var CREATE_REPLICATION_SLOT_TIMEOUT_MS = 3e4;
+var CREATE_REPLICATION_SLOT_TIMEOUT_MS = 6e4;
 var SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1e3;
-async function createReplicationSlot(lc, session, { slotName, failover, temporary, lockTimeout = SERVER_LOCK_TIMEOUT_MS }) {
+/** Thrown when the client-side timeout for creating a slot is reached. */
+var SlotCreationTimeoutError = class extends Error {
+	name = "SlotCreationTimeoutError";
+};
+async function createReplicationSlot(lc, session, { slotName, failover, temporary, lockTimeout = SERVER_LOCK_TIMEOUT_MS, diagnosticsDB }) {
 	await session.unsafe(`SET lock_timeout = ${lockTimeout}`);
+	const { pgVersion, pid: walsenderPID } = (await session.unsafe(`
+      SELECT current_setting('server_version_num') as "pgVersion",
+             pg_backend_pid() as "pid";
+  `))[0];
 	const maybeTemporary = temporary ? "TEMPORARY" : "";
-	const options = failover ? "(FAILOVER)" : "";
-	const raced = await orTimeout(session.unsafe(`
-    CREATE_REPLICATION_SLOT "${slotName}" ${maybeTemporary} LOGICAL pgoutput ${options}`), CREATE_REPLICATION_SLOT_TIMEOUT_MS);
-	if (raced === "timed-out") {
-		session.end().catch((e) => lc.warn?.(`Error closing timed out replication slot session`, e));
-		throw new Error(`Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}. Crashing to force a clean restart.`);
+	const options = failover && pgVersion >= 17e4 ? "(FAILOVER)" : "";
+	const createSlot = session.unsafe(`
+    CREATE_REPLICATION_SLOT "${slotName}" ${maybeTemporary} LOGICAL pgoutput ${options}`);
+	try {
+		const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);
+		if (raced === "timed-out") throw new SlotCreationTimeoutError(`Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`);
+		const [slot] = raced;
+		lc.info?.(`Created replication slot ${slotName}`, slot);
+		return slot;
+	} catch (e) {
+		if (diagnosticsDB && (isPostgresError(e, PG_LOCK_NOT_AVAILABLE) || e instanceof SlotCreationTimeoutError)) await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);
+		throw e;
 	}
-	const [slot] = raced;
-	lc.info?.(`Created replication slot ${slotName}`, slot);
-	return slot;
 }
+var SLOT_DIAGNOSTICS_TIMEOUT_MS = 5e3;
+var SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;
 /**
-* Replica and slot creation involves two sessions for proper coordination
-* with other replica management logic:
+* Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting
+* on: the walsender's wait event and blocking pids, and the oldest
+* transactions holding an xid. Slot creation waits for every such transaction
+* on the instance (in all databases), each with its own lock_timeout, so a
+* chain of overlapping transactions can exceed the client-side timeout
+* without the lock_timeout ever firing.
+*
+* Query text is deliberately omitted since it can contain row values.
 *
-* * A normal transaction is started and acquires an advisory lock for
+* Exported for testing.
+*/
+async function logSlotCreationBlockers(lc, sql, walsenderPID, slotName) {
+	try {
+		const result = await orTimeout(Promise.all([sql`
+          SELECT pid, wait_event_type as "waitEventType", wait_event as "waitEvent",
+                 pg_blocking_pids(pid) as "blockingPIDs"
+            FROM pg_stat_activity WHERE pid = ${walsenderPID}`, sql`
+          SELECT pid, datname, usename, application_name as "applicationName",
+                 backend_type as "backendType", state,
+                 wait_event_type as "waitEventType", wait_event as "waitEvent",
+                 backend_xid::text as xid,
+                 xact_start::text as "xactStart",
+                 extract(epoch from now() - xact_start)::float8 as "xactAgeSec"
+            FROM pg_stat_activity
+            WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}
+            ORDER BY xact_start NULLS LAST
+            LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`]), SLOT_DIAGNOSTICS_TIMEOUT_MS);
+		if (result === "timed-out") {
+			lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);
+			return;
+		}
+		const [[walsender], transactions] = result;
+		lc.warn?.(`Replication slot ${slotName} creation blocked`, {
+			walsender: walsender ?? null,
+			transactions
+		});
+	} catch (e) {
+		lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);
+	}
+}
+/**
+* Whether slot creation failed because it was waiting on transactions on the
+* upstream instance: the server's lock_timeout fired, the client-side timeout
+* was reached, or the replication session was closed while it was waiting.
+* Such failures are expected on busy upstreams, and leave no durable state,
+* so the attempt can simply be retried.
+*/
+function isSlotCreationBlocked(e) {
+	return isPostgresError(e, PG_LOCK_NOT_AVAILABLE) || e instanceof SlotCreationTimeoutError || e instanceof Error && e.code === "CONNECTION_CLOSED";
+}
+var DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS = 1800 * 1e3;
+var DEFAULT_SLOT_CREATION_RETRY_DELAY_MS = 1e3;
+/**
+* Creates a dedicated (non-pooled) session for holding the session-level
+* replication slot management lock. Ending the session releases the lock,
+* which guarantees that the lock never outlives the attempt.
+*/
+function createLockSessionFor(db, applicationName) {
+	return postgres(defu({
+		max: 1,
+		["idle_timeout"]: null,
+		["max_lifetime"]: null,
+		connection: { ["application_name"]: applicationName }
+	}, db.options));
+}
+/**
+* Replica and slot creation involves several sessions for proper
+* coordination with other replica management logic:
+*
+* * A dedicated session acquires a (session-level) advisory lock for
 *   replica slot management. This is the same lock that cleanup logic
-*   acquires before cleaning up replication slots.
+*   acquires before cleaning up replication slots. The lock is held
+*   outside of a transaction so that the session is not "idle in
+*   transaction" while slot creation waits on upstream transactions,
+*   which some upstreams terminate (e.g. idle_in_transaction_session_timeout).
 * * With the lock held, a new replication slot is created in a
 *   replication session. The API of CREATE_REPLICATION_SLOT is such
 *   that it cannot be done in a transaction, and cannot be followed by
@@ -38,6 +127,14 @@ async function createReplicationSlot(lc, session, { slotName, failover, temporar
 * * Once the slot is created, the slot and replica information are recorded
 *   in the `replicas` table before releasing the lock.
 *
+* Slot creation waits for all transactions on the upstream instance that
+* hold an xid, so it can be blocked by long-running transactions. A blocked
+* attempt is abandoned (see {@link CREATE_REPLICATION_SLOT_TIMEOUT_MS}) and
+* retried in-process, as it leaves no durable state: the `replicas` row is
+* only inserted after the slot has been created. Retries continue until the
+* slot is created or until `maxBlockedMs` has elapsed, after which the error
+* is thrown.
+*
 * This locking ensures that:
 * 1. multiple replication managers attempting to create a replication slot
 *    will not use the same name for the replication slot (which is selected
@@ -62,85 +159,186 @@ async function createReplicationSlot(lc, session, { slotName, failover, temporar
 * PostgresChangeSource takes over the slot to process the stream in earnest.
 * It is also returned for cleanup in the case of failures (and for control
 * in unit tests).
+*
+* For replica's created in the `Restore` stage, the `generation` is
+* initialized to an empty string instead of the slot's consistent point,
+* since the slot is not being used for initial sync. It is the
+* responsibiliy of the caller to call {@link initRestoreReplica} when
+* the source replica has been selected for restoring, which will carry
+* over the `generation` and other relevant values from the source.
 */
-async function createReplicaAndSlot(lc, sql, sessionName, shard, replicaID, failover, backupOptions, captureSnapshot) {
-	const replicationSession = createReplicationSessionFor(sql, sessionName);
+async function createReplicaAndSlot(lc, sql, sessionName, shard, epoch, replicaID, failover, backupOptions, captureSnapshot, stage, { lockTimeout, maxBlockedMs = DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS, retryDelayMs = DEFAULT_SLOT_CREATION_RETRY_DELAY_MS } = {}) {
 	const lockName = replicationSlotManagementLock(shard);
 	const slotPoolPrefix = replicationSlotPrefix(shard);
-	for (let first = true;; first = false) {
+	const start = Date.now();
+	let addedReplicationRole = false;
+	let cleanedUpForSlotLimit = false;
+	for (let attempt = 1;; attempt++) {
 		await dropUnclaimedSlots(lc, sql, shard);
+		const replicationSession = createReplicationSessionFor(sql, sessionName);
+		const lockSession = createLockSessionFor(sql, `${sessionName}-lock`);
+		const endSessions = () => Promise.allSettled([replicationSession.end({ timeout: 5 }), lockSession.end({ timeout: 5 })]);
 		let slotName;
+		let initialSession;
+		let creatingSlot = false;
 		try {
-			return await runTx(sql, async (tx) => {
-				await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;
-				const names = await tx`
-          SELECT slot_name as name FROM pg_replication_slots
-            WHERE slot_name LIKE ${slotPoolPrefix + "%"};
-        `.values();
-				const inUse = new Set(names.flat());
-				for (let next = 0;; next++) {
-					const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;
-					if (!inUse.has(candidateName)) {
-						slotName = candidateName;
-						break;
-					}
+			await lockSession`SELECT pg_advisory_lock(hashtext(${lockName}))`;
+			if (stage === 0) {
+				const others = await getRestoreCandidates(sql, shard, epoch);
+				lc.info?.(`current replicas at epoch ${epoch}`, { replicas: others });
+				if (others.length) {
+					const [{ id, stage, active }] = others;
+					if (stage === 0 && active) throw new AutoResetSignal(`another replica (${id}) is performing initial sync`);
+				}
+			}
+			const names = await sql`
+        SELECT slot_name as name FROM pg_replication_slots
+          WHERE slot_name LIKE ${slotPoolPrefix + "%"};
+      `.values();
+			const inUse = new Set(names.flat());
+			for (let next = 0;; next++) {
+				const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;
+				if (!inUse.has(candidateName)) {
+					slotName = candidateName;
+					break;
 				}
-				const slot = await createReplicationSlot(lc, replicationSession, {
-					slotName,
-					failover
-				});
-				const capturedSnapshot = await captureSnapshot(slot.snapshot_name);
-				const initialSession = await keepSlotActiveUntilTakenOver(lc, replicationSession, slot.slot_name, metadataPublicationName(shard.appID, shard.shardNum), toBigInt(slot.consistent_point));
-				await createReplica(tx, shard, replicaID, slot.slot_name, toStateVersionString(slot.consistent_point), backupOptions);
-				return {
-					slot,
-					capturedSnapshot,
-					initialSession
-				};
+			}
+			creatingSlot = true;
+			const slot = await createReplicationSlot(lc, replicationSession, {
+				slotName,
+				failover,
+				lockTimeout,
+				diagnosticsDB: sql
+			});
+			creatingSlot = false;
+			const capturedSnapshot = await captureSnapshot(slot.snapshot_name);
+			await replicationSession.end({ timeout: 5 });
+			initialSession = await keepSlotActiveUntilTakenOver(lc, {
+				db: extractConnectionConfig(sql),
+				slot: slot.slot_name,
+				dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),
+				lsn: String(toBigInt(slot.consistent_point))
 			});
+			const replica = await runTx(sql, async (tx) => {
+				await createReplica(tx, shard, replicaID, slot.slot_name, epoch, stage === 0 ? toStateVersionString(slot.consistent_point) : "", backupOptions, stage);
+				return must(await getReplicaState(tx, shard, replicaID), `replica ${replicaID} was not created`);
+			});
+			return {
+				slot,
+				capturedSnapshot,
+				initialSession,
+				replica
+			};
 		} catch (e) {
-			if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {
+			await endSessions();
+			if (!addedReplicationRole && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {
 				await sql`ALTER ROLE current_user WITH REPLICATION`;
 				lc.info?.(`Added the REPLICATION role to database user`);
+				addedReplicationRole = true;
 				continue;
 			}
-			if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {
+			if (!cleanedUpForSlotLimit && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {
 				lc.warn?.(`Reached max replication slots. Attempting to clean up unused slots`, e);
 				await sql`
           DELETE FROM ${sql(`${upstreamSchema(shard)}.replicas`)} USING pg_replication_slots slots
             WHERE replicas.slot = slots.slot_name AND NOT slots.active`;
+				cleanedUpForSlotLimit = true;
+				continue;
+			}
+			if (initialSession) try {
+				await endSession(initialSession);
+			} catch {}
+			const blockedMs = Date.now() - start;
+			if (creatingSlot && isSlotCreationBlocked(e) && blockedMs < maxBlockedMs) {
+				lc.warn?.(`Replication slot ${slotName} creation blocked by upstream transactions (attempt ${attempt}, ${blockedMs} ms). Retrying.`, e);
+				await sleep(retryDelayMs * (1 + 4 * Math.random()));
 				continue;
 			}
 			if (slotName) {
 				lc.warn?.(`deleting slot ${slotName} due to error`, e);
-				await replicationSession.end();
-				await sql`SELECT pg_drop_replication_slot(${slotName})`;
+				await dropUnclaimedSlots(lc, sql, shard);
 			}
 			throw e;
+		} finally {
+			await endSessions();
 		}
 	}
 }
 /**
-* Deletes "old" replicas (i.e. those with a lower rank than the current)
-* and attempts to drop replication slots that are not associated with any
-* replica.
+* Claims the inactive slot of the given replica for use by this task, opening
+* a replication session to mark it as active. The process should follow up by
+* taking over the slot via the PostgresChangeSource.
 *
-* If a slot could not be dropped because there is still an active subscriber,
-* it will be reflected in the `draining` count that is returned. When there
-* are draining slots, the method should be retried until all orphaned slots
-* have been dropped.
+* The replica is identified by its ID (and not just its slot) because rows of
+* replicas whose slots were dropped outside of zero-cache's control may share
+* the slot name.
 */
-async function dropOldReplicasAndSlots(lc, sql, shard, beforeRank) {
+async function claimSlotForResumption(lc, sql, shard, _sessionName, { id: replicaID, slot }) {
+	const lockName = replicationSlotManagementLock(shard);
 	const replicasTable = `${upstreamSchema(shard)}.replicas`;
-	const oldReplicas = await sql`
-    SELECT id, rank::float8, slot, version, "initialSyncContext", "subscriberContext"
-     FROM ${sql(replicasTable)} WHERE rank < ${beforeRank};
-  `;
-	if (oldReplicas.length) {
-		lc.info?.(`Deleting ${oldReplicas.length} old replica(s)`, { oldReplicas });
-		await sql`DELETE FROM ${sql(replicasTable)} WHERE rank < ${beforeRank}`;
+	let reservation;
+	try {
+		const reserved = await runTx(sql, async (tx) => {
+			await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;
+			const replicas = await tx`
+      SELECT active, confirmed_flush_lsn as lsn
+        FROM pg_replication_slots
+        JOIN ${tx(replicasTable)} replica on slot_name = slot
+        WHERE slot_name = ${slot} AND replica.id = ${replicaID};
+    `;
+			if (replicas.length === 0) {
+				lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);
+				return null;
+			}
+			const [{ active, lsn }] = replicas;
+			if (active) {
+				lc.warn?.(`replica ${replicaID} for slot ${slot} is active and cannot be claimed`);
+				return null;
+			}
+			await tx`
+        UPDATE ${tx(replicasTable)} SET "stage" = ${2} WHERE id = ${replicaID};
+      `;
+			const replica = must(await getReplicaState(tx, shard, replicaID), `replica ${replicaID} disappeared`);
+			reservation = await keepSlotActiveUntilTakenOver(lc, {
+				db: extractConnectionConfig(sql),
+				slot,
+				dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),
+				lsn: String(toBigInt(lsn))
+			});
+			return {
+				slot,
+				replica,
+				reservation
+			};
+		});
+		lc.info?.(`successfully claimed slot ${slot}`);
+		return reserved;
+	} catch (e) {
+		if (reservation) try {
+			await endSession(reservation);
+		} catch {}
+		lc.error?.(`unable to claim replication slot ${slot}`, e);
+		return null;
 	}
-	return dropUnclaimedSlots(lc, sql, shard);
+}
+function dropInactiveSlotsAndReplicas(lc, sql, shard, slots) {
+	const lockName = replicationSlotManagementLock(shard);
+	const replicasTable = `${upstreamSchema(shard)}.replicas`;
+	return runTx(sql, async (tx) => {
+		await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;
+		const dropped = await tx`
+      SELECT slot_name as slot, replica.id as "replicaID", pg_drop_replication_slot(slot_name) 
+        FROM pg_replication_slots
+        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot
+        WHERE slot_name IN ${tx(slots)} AND NOT active;
+    `;
+		if (dropped.length) lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, { dropped });
+		const replicas = dropped.map(({ replicaID }) => replicaID).filter((id) => id !== null);
+		if (replicas.length) {
+			await tx`DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;
+			lc.info?.(`deleted ${replicas.length} old replica(s)`, { replicas });
+		}
+	});
 }
 function dropUnclaimedSlots(lc, sql, shard) {
 	const lockName = replicationSlotManagementLock(shard);
@@ -157,6 +355,14 @@ function dropUnclaimedSlots(lc, sql, shard) {
           AND replica.id IS NULL;
     `;
 		if (dropped.length) lc.info?.(`dropped inactive replication slots`, { dropped });
+		const orphaned = await tx`
+      DELETE FROM ${tx(replicasTable)} replica
+        WHERE NOT EXISTS (
+          SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot
+        )
+        RETURNING id;
+    `;
+		if (orphaned.length) lc.warn?.(`deleted replicas whose slots no longer exist`, { replicas: orphaned.map(({ id }) => id) });
 		const remaining = await tx`
       SELECT slot_name as slot, active_pid as pid, replica.id as id
         FROM pg_replication_slots
@@ -190,6 +396,6 @@ function replicationSlotManagementLock(shard) {
 	return `replication-slot-management:${shard.appID}_${shard.shardNum}`;
 }
 //#endregion
-export { createReplicaAndSlot, createReplicationSlot, dropOldReplicasAndSlots };
+export { claimSlotForResumption, createReplicaAndSlot, createReplicationSlot, dropInactiveSlotsAndReplicas };
 
 //# sourceMappingURL=replication-slots.js.map