@rocicorp/zero 1.11.0-canary.34 → 1.11.0-canary.36

package/out/zero-cache/src/services/change-source/pg/replication-slots.js

@@ -1,3 +1,4 @@
+import { sleep } from "../../../../../shared/src/sleep.js";
 import { must } from "../../../../../shared/src/must.js";
 import { upstreamSchema } from "../../../types/shards.js";
 import { isPostgresError } from "../../../types/pg.js";
@@ -9,10 +10,16 @@ import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
 import "../../../types/pg-versions.js";
 import { createReplicationSessionFor, endSession, extractConnectionConfig, keepSlotActiveUntilTakenOver } from "./logical-replication/stream.js";
 import { orTimeout } from "../../../types/timeout.js";
-import { PG_CONFIGURATION_LIMIT_EXCEEDED, PG_INSUFFICIENT_PRIVILEGE } from "@drdgvhbh/postgres-error-codes";
+import { defu } from "defu";
+import postgres from "postgres";
+import { PG_CONFIGURATION_LIMIT_EXCEEDED, PG_INSUFFICIENT_PRIVILEGE, PG_LOCK_NOT_AVAILABLE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/replication-slots.ts
 var CREATE_REPLICATION_SLOT_TIMEOUT_MS = 6e4;
 var SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1e3;
+/** Thrown when the client-side timeout for creating a slot is reached. */
+var SlotCreationTimeoutError = class extends Error {
+	name = "SlotCreationTimeoutError";
+};
 async function createReplicationSlot(lc, session, { slotName, failover, temporary, lockTimeout = SERVER_LOCK_TIMEOUT_MS, diagnosticsDB }) {
 	await session.unsafe(`SET lock_timeout = ${lockTimeout}`);
 	const { pgVersion, pid: walsenderPID } = (await session.unsafe(`
@@ -21,16 +28,18 @@ async function createReplicationSlot(lc, session, { slotName, failover, temporar
   `))[0];
 	const maybeTemporary = temporary ? "TEMPORARY" : "";
 	const options = failover && pgVersion >= 17e4 ? "(FAILOVER)" : "";
-	const raced = await orTimeout(session.unsafe(`
-    CREATE_REPLICATION_SLOT "${slotName}" ${maybeTemporary} LOGICAL pgoutput ${options}`), CREATE_REPLICATION_SLOT_TIMEOUT_MS);
-	if (raced === "timed-out") {
-		if (diagnosticsDB) await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);
-		session.end().catch((e) => lc.warn?.(`Error closing timed out replication slot session`, e));
-		throw new Error(`Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`);
+	const createSlot = session.unsafe(`
+    CREATE_REPLICATION_SLOT "${slotName}" ${maybeTemporary} LOGICAL pgoutput ${options}`);
+	try {
+		const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);
+		if (raced === "timed-out") throw new SlotCreationTimeoutError(`Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`);
+		const [slot] = raced;
+		lc.info?.(`Created replication slot ${slotName}`, slot);
+		return slot;
+	} catch (e) {
+		if (diagnosticsDB && (isPostgresError(e, PG_LOCK_NOT_AVAILABLE) || e instanceof SlotCreationTimeoutError)) await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);
+		throw e;
 	}
-	const [slot] = raced;
-	lc.info?.(`Created replication slot ${slotName}`, slot);
-	return slot;
 }
 var SLOT_DIAGNOSTICS_TIMEOUT_MS = 5e3;
 var SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;
@@ -76,12 +85,40 @@ async function logSlotCreationBlockers(lc, sql, walsenderPID, slotName) {
 	}
 }
 /**
-* Replica and slot creation involves two sessions for proper coordination
-* with other replica management logic:
+* Whether slot creation failed because it was waiting on transactions on the
+* upstream instance: the server's lock_timeout fired, the client-side timeout
+* was reached, or the replication session was closed while it was waiting.
+* Such failures are expected on busy upstreams, and leave no durable state,
+* so the attempt can simply be retried.
+*/
+function isSlotCreationBlocked(e) {
+	return isPostgresError(e, PG_LOCK_NOT_AVAILABLE) || e instanceof SlotCreationTimeoutError || e instanceof Error && e.code === "CONNECTION_CLOSED";
+}
+var DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS = 1800 * 1e3;
+var DEFAULT_SLOT_CREATION_RETRY_DELAY_MS = 1e3;
+/**
+* Creates a dedicated (non-pooled) session for holding the session-level
+* replication slot management lock. Ending the session releases the lock,
+* which guarantees that the lock never outlives the attempt.
+*/
+function createLockSessionFor(db, applicationName) {
+	return postgres(defu({
+		max: 1,
+		["idle_timeout"]: null,
+		["max_lifetime"]: null,
+		connection: { ["application_name"]: applicationName }
+	}, db.options));
+}
+/**
+* Replica and slot creation involves several sessions for proper
+* coordination with other replica management logic:
 *
-* * A normal transaction is started and acquires an advisory lock for
+* * A dedicated session acquires a (session-level) advisory lock for
 *   replica slot management. This is the same lock that cleanup logic
-*   acquires before cleaning up replication slots.
+*   acquires before cleaning up replication slots. The lock is held
+*   outside of a transaction so that the session is not "idle in
+*   transaction" while slot creation waits on upstream transactions,
+*   which some upstreams terminate (e.g. idle_in_transaction_session_timeout).
 * * With the lock held, a new replication slot is created in a
 *   replication session. The API of CREATE_REPLICATION_SLOT is such
 *   that it cannot be done in a transaction, and cannot be followed by
@@ -90,6 +127,14 @@ async function logSlotCreationBlockers(lc, sql, walsenderPID, slotName) {
 * * Once the slot is created, the slot and replica information are recorded
 *   in the `replicas` table before releasing the lock.
 *
+* Slot creation waits for all transactions on the upstream instance that
+* hold an xid, so it can be blocked by long-running transactions. A blocked
+* attempt is abandoned (see {@link CREATE_REPLICATION_SLOT_TIMEOUT_MS}) and
+* retried in-process, as it leaves no durable state: the `replicas` row is
+* only inserted after the slot has been created. Retries continue until the
+* slot is created or until `maxBlockedMs` has elapsed, after which the error
+* is thrown.
+*
 * This locking ensures that:
 * 1. multiple replication managers attempting to create a replication slot
 *    will not use the same name for the replication slot (which is selected
@@ -122,83 +167,100 @@ async function logSlotCreationBlockers(lc, sql, walsenderPID, slotName) {
 * the source replica has been selected for restoring, which will carry
 * over the `generation` and other relevant values from the source.
 */
-async function createReplicaAndSlot(lc, sql, sessionName, shard, epoch, replicaID, failover, backupOptions, captureSnapshot, stage) {
-	const replicationSession = createReplicationSessionFor(sql, sessionName);
+async function createReplicaAndSlot(lc, sql, sessionName, shard, epoch, replicaID, failover, backupOptions, captureSnapshot, stage, { lockTimeout, maxBlockedMs = DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS, retryDelayMs = DEFAULT_SLOT_CREATION_RETRY_DELAY_MS } = {}) {
 	const lockName = replicationSlotManagementLock(shard);
 	const slotPoolPrefix = replicationSlotPrefix(shard);
-	for (let first = true;; first = false) {
+	const start = Date.now();
+	let addedReplicationRole = false;
+	let cleanedUpForSlotLimit = false;
+	for (let attempt = 1;; attempt++) {
 		await dropUnclaimedSlots(lc, sql, shard);
+		const replicationSession = createReplicationSessionFor(sql, sessionName);
+		const lockSession = createLockSessionFor(sql, `${sessionName}-lock`);
+		const endSessions = () => Promise.allSettled([replicationSession.end({ timeout: 5 }), lockSession.end({ timeout: 5 })]);
 		let slotName;
 		let initialSession;
+		let creatingSlot = false;
 		try {
-			return await runTx(sql, async (tx) => {
-				await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;
-				if (stage === 0) {
-					const others = await getRestoreCandidates(tx, shard, epoch);
-					lc.info?.(`current replicas at epoch ${epoch}`, { replicas: others });
-					if (others.length) {
-						const [{ id, stage, active }] = others;
-						if (stage === 0 && active) throw new AutoResetSignal(`another replica (${id}) is performing initial sync`);
-					}
+			await lockSession`SELECT pg_advisory_lock(hashtext(${lockName}))`;
+			if (stage === 0) {
+				const others = await getRestoreCandidates(sql, shard, epoch);
+				lc.info?.(`current replicas at epoch ${epoch}`, { replicas: others });
+				if (others.length) {
+					const [{ id, stage, active }] = others;
+					if (stage === 0 && active) throw new AutoResetSignal(`another replica (${id}) is performing initial sync`);
 				}
-				const names = await tx`
-          SELECT slot_name as name FROM pg_replication_slots
-            WHERE slot_name LIKE ${slotPoolPrefix + "%"};
-        `.values();
-				const inUse = new Set(names.flat());
-				for (let next = 0;; next++) {
-					const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;
-					if (!inUse.has(candidateName)) {
-						slotName = candidateName;
-						break;
-					}
+			}
+			const names = await sql`
+        SELECT slot_name as name FROM pg_replication_slots
+          WHERE slot_name LIKE ${slotPoolPrefix + "%"};
+      `.values();
+			const inUse = new Set(names.flat());
+			for (let next = 0;; next++) {
+				const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;
+				if (!inUse.has(candidateName)) {
+					slotName = candidateName;
+					break;
 				}
-				const slot = await createReplicationSlot(lc, replicationSession, {
-					slotName,
-					failover,
-					diagnosticsDB: sql
-				});
-				const capturedSnapshot = await captureSnapshot(slot.snapshot_name);
-				await replicationSession.end();
-				initialSession = await keepSlotActiveUntilTakenOver(lc, {
-					db: extractConnectionConfig(sql),
-					slot: slot.slot_name,
-					dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),
-					lsn: String(toBigInt(slot.consistent_point))
-				});
+			}
+			creatingSlot = true;
+			const slot = await createReplicationSlot(lc, replicationSession, {
+				slotName,
+				failover,
+				lockTimeout,
+				diagnosticsDB: sql
+			});
+			creatingSlot = false;
+			const capturedSnapshot = await captureSnapshot(slot.snapshot_name);
+			await replicationSession.end({ timeout: 5 });
+			initialSession = await keepSlotActiveUntilTakenOver(lc, {
+				db: extractConnectionConfig(sql),
+				slot: slot.slot_name,
+				dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),
+				lsn: String(toBigInt(slot.consistent_point))
+			});
+			const replica = await runTx(sql, async (tx) => {
 				await createReplica(tx, shard, replicaID, slot.slot_name, epoch, stage === 0 ? toStateVersionString(slot.consistent_point) : "", backupOptions, stage);
-				const replica = must(await getReplicaState(tx, shard, replicaID), `replica ${replicaID} was not created`);
-				return {
-					slot,
-					capturedSnapshot,
-					initialSession,
-					replica
-				};
+				return must(await getReplicaState(tx, shard, replicaID), `replica ${replicaID} was not created`);
 			});
+			return {
+				slot,
+				capturedSnapshot,
+				initialSession,
+				replica
+			};
 		} catch (e) {
-			if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {
+			await endSessions();
+			if (!addedReplicationRole && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {
 				await sql`ALTER ROLE current_user WITH REPLICATION`;
 				lc.info?.(`Added the REPLICATION role to database user`);
+				addedReplicationRole = true;
 				continue;
 			}
-			if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {
+			if (!cleanedUpForSlotLimit && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {
 				lc.warn?.(`Reached max replication slots. Attempting to clean up unused slots`, e);
 				await sql`
           DELETE FROM ${sql(`${upstreamSchema(shard)}.replicas`)} USING pg_replication_slots slots
             WHERE replicas.slot = slots.slot_name AND NOT slots.active`;
+				cleanedUpForSlotLimit = true;
 				continue;
 			}
 			if (initialSession) try {
 				await endSession(initialSession);
 			} catch {}
+			const blockedMs = Date.now() - start;
+			if (creatingSlot && isSlotCreationBlocked(e) && blockedMs < maxBlockedMs) {
+				lc.warn?.(`Replication slot ${slotName} creation blocked by upstream transactions (attempt ${attempt}, ${blockedMs} ms). Retrying.`, e);
+				await sleep(retryDelayMs * (1 + 4 * Math.random()));
+				continue;
+			}
 			if (slotName) {
 				lc.warn?.(`deleting slot ${slotName} due to error`, e);
-				try {
-					await replicationSession.end();
-				} catch {}
 				await dropUnclaimedSlots(lc, sql, shard);
 			}
 			throw e;
+		} finally {
+			await endSessions();
 		}
 	}
 }

package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type postgres from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  createReplicationSessionFor,\n  endSession,\n  extractConnectionConfig,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  InitialSync,\n  Restore,\n  type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n  createReplica,\n  getReplicaState,\n  getRestoreCandidates,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n  type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n  replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n  slot: string;\n  reservation: Readable;\n  replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: ignored if pgVersion < PG_17.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number;\n\n  // A (non-replication) connection to the same upstream instance, used to\n  // log what slot creation was waiting on if it times out.\n  diagnosticsDB?: PostgresDB | undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n    diagnosticsDB,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const {pgVersion, pid: walsenderPID} = (\n    await session.unsafe<{pgVersion: number; pid: number}[]>(`\n      SELECT current_setting('server_version_num') as \"pgVersion\",\n             pg_backend_pid() as \"pid\";\n  `)\n  )[0];\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n  if (raced === 'timed-out') {\n    if (diagnosticsDB) {\n      await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n    }\n    // Create slot can block indefinitely waiting for old transactions. End\n    // this connection in the background and fail fast so the process restarts.\n    void session\n      .end()\n      .catch(e =>\n        lc.warn?.(`Error closing timed out replication slot session`, e),\n      );\n    throw new Error(\n      `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n    );\n  }\n  const [slot] = raced;\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/**\n * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n *\n * Query text is deliberately omitted since it can contain row values.\n *\n * Exported for testing.\n */\nexport async function logSlotCreationBlockers(\n  lc: LogContext,\n  sql: PostgresDB,\n  walsenderPID: number,\n  slotName: string,\n) {\n  try {\n    const result = await orTimeout(\n      Promise.all([\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 pg_blocking_pids(pid) as \"blockingPIDs\"\n            FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, datname, usename, application_name as \"applicationName\",\n                 backend_type as \"backendType\", state,\n                 wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 backend_xid::text as xid,\n                 xact_start::text as \"xactStart\",\n                 extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n            FROM pg_stat_activity\n            WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n            ORDER BY xact_start NULLS LAST\n            LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n      ]),\n      SLOT_DIAGNOSTICS_TIMEOUT_MS,\n    );\n    if (result === 'timed-out') {\n      lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n      return;\n    }\n    const [[walsender], transactions] = result;\n    lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n      walsender: walsender ?? null,\n      transactions,\n    });\n  } catch (e) {\n    lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n  }\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  epoch: number,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n  stage: ReplicaStage,\n): Promise<ReplicationSlotResult<T>> {\n  // Note: The replicationSession is used to create the replication slot\n  // and closed immediately afterwards, or on error.\n  const replicationSession = createReplicationSessionFor(sql, sessionName);\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n\n  for (let first = true; ; first = false) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    let slotName: string | undefined;\n    let initialSession: Readable | undefined;\n    try {\n      return await runTx(sql, async tx => {\n        await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n        if (stage === InitialSync) {\n          // With the lock acquired, ensure that only one initial sync is\n          // active at a time. getRestoreCandidates() orders its results\n          // with stage = InitialSync (and active = true) first.\n          const others = await getRestoreCandidates(tx, shard, epoch);\n          lc.info?.(`current replicas at epoch ${epoch}`, {replicas: others});\n          if (others.length) {\n            const [{id, stage, active}] = others;\n            if (stage === InitialSync && active) {\n              throw new AutoResetSignal(\n                `another replica (${id}) is performing initial sync`,\n              );\n            }\n          }\n        }\n\n        // Pick an available slotName from the slotPoolPrefix pool.\n        const names = await tx<{name: string}[]> /*sql*/ `\n          SELECT slot_name as name FROM pg_replication_slots\n            WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n        `.values();\n        const inUse = new Set(names.flat());\n        for (let next = 0; ; next++) {\n          const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n          if (!inUse.has(candidateName)) {\n            slotName = candidateName;\n            break;\n          }\n        }\n\n        const slot = await createReplicationSlot(lc, replicationSession, {\n          slotName,\n          failover,\n          diagnosticsDB: sql,\n        });\n        const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n        await replicationSession.end();\n        initialSession = await keepSlotActiveUntilTakenOver(lc, {\n          db: extractConnectionConfig(sql),\n          slot: slot.slot_name,\n          dummyPublication: metadataPublicationName(\n            shard.appID,\n            shard.shardNum,\n          ),\n          lsn: String(toBigInt(slot.consistent_point)),\n        });\n\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          epoch,\n          stage === InitialSync\n            ? toStateVersionString(slot.consistent_point)\n            : '', // initialized with initRestoreReplica\n          backupOptions,\n          stage,\n        );\n\n        const replica = must(\n          await getReplicaState(tx, shard, replicaID),\n          `replica ${replicaID} was not created`,\n        );\n\n        return {slot, capturedSnapshot, initialSession, replica};\n      });\n    } catch (e) {\n      if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      if (initialSession) {\n        try {\n          await endSession(initialSession);\n        } catch {}\n      }\n      // Otherwise, clean up any created slot if something went wrong. The\n      // slot may not exist (e.g. if its creation failed), and its name may\n      // have been reused since the lock was released, so leave it to\n      // dropUnclaimedSlots(), which only drops slots without a replica.\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        try {\n          await replicationSession.end();\n        } catch {}\n        await dropUnclaimedSlots(lc, sql, shard);\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n *\n * The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n */\nexport async function claimSlotForResumption(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  _sessionName: string,\n  {id: replicaID, slot}: Pick<ReplicaState, 'id' | 'slot'>,\n): Promise<ReservedSlot | null> {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  let reservation: Readable | undefined;\n  try {\n    const reserved = await runTx(sql, async tx => {\n      await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n      const replicas = await tx<{active: boolean; lsn: string}[]> /*sql*/ `\n      SELECT active, confirmed_flush_lsn as lsn\n        FROM pg_replication_slots\n        JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n    `;\n      if (replicas.length === 0) {\n        lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n        return null;\n      }\n      const [{active, lsn}] = replicas;\n      if (active) {\n        lc.warn?.(\n          `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n        );\n        return null;\n      }\n      // Mark the replica as Restoring.\n      await tx`\n        UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n      `;\n      const replica = must(\n        await getReplicaState(tx, shard, replicaID),\n        `replica ${replicaID} disappeared`,\n      );\n\n      reservation = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(lsn)),\n      });\n      return {\n        slot,\n        replica,\n        reservation,\n      };\n    });\n    lc.info?.(`successfully claimed slot ${slot}`);\n    return reserved;\n  } catch (e) {\n    if (reservation) {\n      try {\n        await endSession(reservation);\n      } catch {}\n    }\n    lc.error?.(`unable to claim replication slot ${slot}`, e);\n    return null;\n  }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  slots: string[],\n) {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx<{slot: string; replicaID: string | null}[]>\n    /*sql*/ `\n      SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name IN ${tx(slots)} AND NOT active;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n        dropped,\n      });\n    }\n\n    const replicas = dropped\n      .map(({replicaID}) => replicaID)\n      .filter(id => id !== null);\n    if (replicas.length) {\n      // Delete replicas associated with the inactive (and now dropped) slots.\n      await tx\n      /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n      lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n    }\n  });\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    // Conversely, delete replicas whose slots no longer exist, e.g. because\n    // they were dropped outside of zero-cache's control (such as in an\n    // upstream failover). Slot names are reused, so such rows would otherwise\n    // be associated with a new slot of the same name. This is done (and\n    // committed) before the slot is created, both so that the rows are never\n    // associated with it, and because CREATE_REPLICATION_SLOT waits for\n    // transactions that have written (i.e. been assigned an xid).\n    const orphaned = await tx<{id: string}[]> /*sql*/ `\n      DELETE FROM ${tx(replicasTable)} replica\n        WHERE NOT EXISTS (\n          SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n        )\n        RETURNING id;\n    `;\n    if (orphaned.length) {\n      lc.warn?.(`deleted replicas whose slots no longer exist`, {\n        replicas: orphaned.map(({id}) => id),\n      });\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;AAqFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,GACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAIhE,MAAM,QAAQ,MAAM,UAHD,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAE/C,GAAY,kCAAkC;CAC5E,IAAI,UAAU,aAAa;EACzB,IAAI,eACF,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAIzE,QACG,IAAI,EACJ,OAAM,MACL,GAAG,OAAO,oDAAoD,CAAC,CACjE;EACF,MAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;CACF;CACA,MAAM,CAAC,QAAQ;CACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;CACtD,OAAO;AACT;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACmC;CAGnC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAElD,KAAK,IAAI,QAAQ,OAAQ,QAAQ,OAAO;EACtC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAEvC,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,KAAK,OAAM,OAAM;IAClC,MAAM,EAAE,yCAAyC,SAAS;IAE1D,IAAI,UAAA,GAAuB;KAIzB,MAAM,SAAS,MAAM,qBAAqB,IAAI,OAAO,KAAK;KAC1D,GAAG,OAAO,6BAA6B,SAAS,EAAC,UAAU,OAAM,CAAC;KAClE,IAAI,OAAO,QAAQ;MACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;MAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;KAEJ;IACF;IAGA,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,OAAO;IACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;IAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;KAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;KAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;MAC7B,WAAW;MACX;KACF;IACF;IAEA,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;KAC/D;KACA;KACA,eAAe;IACjB,CAAC;IACD,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;IACjE,MAAM,mBAAmB,IAAI;IAC7B,iBAAiB,MAAM,6BAA6B,IAAI;KACtD,IAAI,wBAAwB,GAAG;KAC/B,MAAM,KAAK;KACX,kBAAkB,wBAChB,MAAM,OACN,MAAM,QACR;KACA,KAAK,OAAO,SAAS,KAAK,gBAAgB,CAAC;IAC7C,CAAC;IAED,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IAEA,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBACvB;IAEA,OAAO;KAAC;KAAM;KAAkB;KAAgB;IAAO;GACzD,CAAC;EACH,SAAS,GAAG;GACV,IAAI,SAAS,gBAAgB,GAAG,yBAAyB,GAAG;IAK1D,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD;GACF;GAIA,IAAI,SAAS,gBAAgB,GAAG,+BAA+B,GAAG;IAChE,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC;GACF;GACA,IAAI,gBACF,IAAI;IACF,MAAM,WAAW,cAAc;GACjC,QAAQ,CAAC;GAMX,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,IAAI;KACF,MAAM,mBAAmB,IAAI;IAC/B,QAAQ,CAAC;IACT,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,cACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;CACJ,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAE9E,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aACvB;GAEA,cAAc,MAAM,6BAA6B,IAAI;IACnD,IAAI,wBAAwB,GAAG;IAC/B;IACA,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,GAAG,CAAC;GAC3B,CAAC;GACD,OAAO;IACL;IACA;IACA;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aACF,IAAI;GACF,MAAM,WAAW,WAAW;EAC9B,QAAQ,CAAC;EAEX,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,EAC9B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}
+{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n  PG_LOCK_NOT_AVAILABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  createReplicationSessionFor,\n  endSession,\n  extractConnectionConfig,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  InitialSync,\n  Restore,\n  type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n  createReplica,\n  getReplicaState,\n  getRestoreCandidates,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n  type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n  replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n  slot: string;\n  reservation: Readable;\n  replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: ignored if pgVersion < PG_17.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number | undefined;\n\n  // A (non-replication) connection to the same upstream instance, used to\n  // log what slot creation was waiting on if it times out.\n  diagnosticsDB?: PostgresDB | undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n/** Thrown when the client-side timeout for creating a slot is reached. */\nexport class SlotCreationTimeoutError extends Error {\n  readonly name = 'SlotCreationTimeoutError';\n}\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n    diagnosticsDB,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const {pgVersion, pid: walsenderPID} = (\n    await session.unsafe<{pgVersion: number; pid: number}[]>(`\n      SELECT current_setting('server_version_num') as \"pgVersion\",\n             pg_backend_pid() as \"pid\";\n  `)\n  )[0];\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  try {\n    const raced = await orTimeout(\n      createSlot,\n      CREATE_REPLICATION_SLOT_TIMEOUT_MS,\n    );\n    if (raced === 'timed-out') {\n      throw new SlotCreationTimeoutError(\n        `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n      );\n    }\n    const [slot] = raced;\n    lc.info?.(`Created replication slot ${slotName}`, slot);\n    return slot;\n  } catch (e) {\n    if (\n      diagnosticsDB &&\n      (isPostgresError(e, PG_LOCK_NOT_AVAILABLE) ||\n        e instanceof SlotCreationTimeoutError)\n    ) {\n      // After a lock_timeout, the walsender is no longer waiting, but the\n      // transactions it was waiting on are likely still running. After the\n      // client-side timeout, it may still be waiting on them.\n      await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n    }\n    throw e;\n  }\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/**\n * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n *\n * Query text is deliberately omitted since it can contain row values.\n *\n * Exported for testing.\n */\nexport async function logSlotCreationBlockers(\n  lc: LogContext,\n  sql: PostgresDB,\n  walsenderPID: number,\n  slotName: string,\n) {\n  try {\n    const result = await orTimeout(\n      Promise.all([\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 pg_blocking_pids(pid) as \"blockingPIDs\"\n            FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, datname, usename, application_name as \"applicationName\",\n                 backend_type as \"backendType\", state,\n                 wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 backend_xid::text as xid,\n                 xact_start::text as \"xactStart\",\n                 extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n            FROM pg_stat_activity\n            WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n            ORDER BY xact_start NULLS LAST\n            LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n      ]),\n      SLOT_DIAGNOSTICS_TIMEOUT_MS,\n    );\n    if (result === 'timed-out') {\n      lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n      return;\n    }\n    const [[walsender], transactions] = result;\n    lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n      walsender: walsender ?? null,\n      transactions,\n    });\n  } catch (e) {\n    lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n  }\n}\n\n/**\n * Whether slot creation failed because it was waiting on transactions on the\n * upstream instance: the server's lock_timeout fired, the client-side timeout\n * was reached, or the replication session was closed while it was waiting.\n * Such failures are expected on busy upstreams, and leave no durable state,\n * so the attempt can simply be retried.\n */\nfunction isSlotCreationBlocked(e: unknown): boolean {\n  return (\n    isPostgresError(e, PG_LOCK_NOT_AVAILABLE) ||\n    e instanceof SlotCreationTimeoutError ||\n    (e instanceof Error && (e as {code?: unknown}).code === 'CONNECTION_CLOSED')\n  );\n}\n\n// Slot creation is retried while it is blocked by upstream transactions, but\n// a slot that cannot be created for this long is surfaced as an error (i.e.\n// by exiting the process).\nconst DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS = 30 * 60 * 1000;\n\n// Base delay between retries of blocked slot creation. The actual delay is\n// jittered to between 1x and 5x of this value.\nconst DEFAULT_SLOT_CREATION_RETRY_DELAY_MS = 1_000;\n\nexport type CreateReplicaAndSlotOptions = {\n  // For overriding in tests.\n  lockTimeout?: number | undefined;\n  maxBlockedMs?: number | undefined;\n  retryDelayMs?: number | undefined;\n};\n\n/**\n * Creates a dedicated (non-pooled) session for holding the session-level\n * replication slot management lock. Ending the session releases the lock,\n * which guarantees that the lock never outlives the attempt.\n */\nfunction createLockSessionFor(db: PostgresDB, applicationName: string) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {['application_name']: applicationName},\n      },\n      // See createReplicationSessionFor() for why this cast is necessary.\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\n/**\n * Replica and slot creation involves several sessions for proper\n * coordination with other replica management logic:\n *\n * * A dedicated session acquires a (session-level) advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots. The lock is held\n *   outside of a transaction so that the session is not \"idle in\n *   transaction\" while slot creation waits on upstream transactions,\n *   which some upstreams terminate (e.g. idle_in_transaction_session_timeout).\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * Slot creation waits for all transactions on the upstream instance that\n * hold an xid, so it can be blocked by long-running transactions. A blocked\n * attempt is abandoned (see {@link CREATE_REPLICATION_SLOT_TIMEOUT_MS}) and\n * retried in-process, as it leaves no durable state: the `replicas` row is\n * only inserted after the slot has been created. Retries continue until the\n * slot is created or until `maxBlockedMs` has elapsed, after which the error\n * is thrown.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  epoch: number,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n  stage: ReplicaStage,\n  {\n    lockTimeout,\n    maxBlockedMs = DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS,\n    retryDelayMs = DEFAULT_SLOT_CREATION_RETRY_DELAY_MS,\n  }: CreateReplicaAndSlotOptions = {},\n): Promise<ReplicationSlotResult<T>> {\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n  const start = Date.now();\n  let addedReplicationRole = false;\n  let cleanedUpForSlotLimit = false;\n\n  for (let attempt = 1; ; attempt++) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    // Note: The replicationSession is used to create the replication slot\n    // and closed immediately afterwards, or on error. A new session is used\n    // for each attempt, since a failed attempt closes it.\n    const replicationSession = createReplicationSessionFor(sql, sessionName);\n    const lockSession = createLockSessionFor(sql, `${sessionName}-lock`);\n    // Ending the lock session releases the replication slot management lock.\n    // The sessions are ended with a timeout in case a hung slot creation\n    // (e.g. after a network partition) is still pending.\n    const endSessions = () =>\n      Promise.allSettled([\n        replicationSession.end({timeout: 5}),\n        lockSession.end({timeout: 5}),\n      ]);\n\n    let slotName: string | undefined;\n    let initialSession: Readable | undefined;\n    let creatingSlot = false;\n    try {\n      await lockSession`SELECT pg_advisory_lock(hashtext(${lockName}))`;\n\n      if (stage === InitialSync) {\n        // With the lock acquired, ensure that only one initial sync is\n        // active at a time. getRestoreCandidates() orders its results\n        // with stage = InitialSync (and active = true) first.\n        const others = await getRestoreCandidates(sql, shard, epoch);\n        lc.info?.(`current replicas at epoch ${epoch}`, {replicas: others});\n        if (others.length) {\n          const [{id, stage, active}] = others;\n          if (stage === InitialSync && active) {\n            throw new AutoResetSignal(\n              `another replica (${id}) is performing initial sync`,\n            );\n          }\n        }\n      }\n\n      // Pick an available slotName from the slotPoolPrefix pool.\n      const names = await sql<{name: string}[]> /*sql*/ `\n        SELECT slot_name as name FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n      `.values();\n      const inUse = new Set(names.flat());\n      for (let next = 0; ; next++) {\n        const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n        if (!inUse.has(candidateName)) {\n          slotName = candidateName;\n          break;\n        }\n      }\n\n      creatingSlot = true;\n      const slot = await createReplicationSlot(lc, replicationSession, {\n        slotName,\n        failover,\n        lockTimeout,\n        diagnosticsDB: sql,\n      });\n      creatingSlot = false;\n      const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n      await replicationSession.end({timeout: 5});\n      initialSession = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot: slot.slot_name,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(slot.consistent_point)),\n      });\n\n      const replica = await runTx(sql, async tx => {\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          epoch,\n          stage === InitialSync\n            ? toStateVersionString(slot.consistent_point)\n            : '', // initialized with initRestoreReplica\n          backupOptions,\n          stage,\n        );\n        return must(\n          await getReplicaState(tx, shard, replicaID),\n          `replica ${replicaID} was not created`,\n        );\n      });\n\n      return {slot, capturedSnapshot, initialSession, replica};\n    } catch (e) {\n      // Release the lock (and end the attempt's sessions) before handling the\n      // error, e.g. as dropUnclaimedSlots() acquires the lock from a different\n      // session.\n      await endSessions();\n      if (\n        !addedReplicationRole &&\n        isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)\n      ) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        addedReplicationRole = true;\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (\n        !cleanedUpForSlotLimit &&\n        isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)\n      ) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        cleanedUpForSlotLimit = true;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      if (initialSession) {\n        try {\n          await endSession(initialSession);\n        } catch {}\n      }\n      // The slot may not exist (e.g. if its creation failed), and its name\n      // may be reused once the lock is released, so any created slot is\n      // left to dropUnclaimedSlots(), which only drops slots without a\n      // replica. This runs at the start of the next attempt.\n      const blockedMs = Date.now() - start;\n      if (\n        creatingSlot &&\n        isSlotCreationBlocked(e) &&\n        blockedMs < maxBlockedMs\n      ) {\n        lc.warn?.(\n          `Replication slot ${slotName} creation blocked by upstream transactions ` +\n            `(attempt ${attempt}, ${blockedMs} ms). Retrying.`,\n          e,\n        );\n        await sleep(retryDelayMs * (1 + 4 * Math.random()));\n        continue;\n      }\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await dropUnclaimedSlots(lc, sql, shard);\n      }\n      throw e;\n    } finally {\n      await endSessions();\n    }\n  }\n}\n\n/**\n * Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n *\n * The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n */\nexport async function claimSlotForResumption(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  _sessionName: string,\n  {id: replicaID, slot}: Pick<ReplicaState, 'id' | 'slot'>,\n): Promise<ReservedSlot | null> {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  let reservation: Readable | undefined;\n  try {\n    const reserved = await runTx(sql, async tx => {\n      await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n      const replicas = await tx<{active: boolean; lsn: string}[]> /*sql*/ `\n      SELECT active, confirmed_flush_lsn as lsn\n        FROM pg_replication_slots\n        JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n    `;\n      if (replicas.length === 0) {\n        lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n        return null;\n      }\n      const [{active, lsn}] = replicas;\n      if (active) {\n        lc.warn?.(\n          `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n        );\n        return null;\n      }\n      // Mark the replica as Restoring.\n      await tx`\n        UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n      `;\n      const replica = must(\n        await getReplicaState(tx, shard, replicaID),\n        `replica ${replicaID} disappeared`,\n      );\n\n      reservation = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(lsn)),\n      });\n      return {\n        slot,\n        replica,\n        reservation,\n      };\n    });\n    lc.info?.(`successfully claimed slot ${slot}`);\n    return reserved;\n  } catch (e) {\n    if (reservation) {\n      try {\n        await endSession(reservation);\n      } catch {}\n    }\n    lc.error?.(`unable to claim replication slot ${slot}`, e);\n    return null;\n  }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  slots: string[],\n) {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx<{slot: string; replicaID: string | null}[]>\n    /*sql*/ `\n      SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name IN ${tx(slots)} AND NOT active;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n        dropped,\n      });\n    }\n\n    const replicas = dropped\n      .map(({replicaID}) => replicaID)\n      .filter(id => id !== null);\n    if (replicas.length) {\n      // Delete replicas associated with the inactive (and now dropped) slots.\n      await tx\n      /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n      lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n    }\n  });\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    // Conversely, delete replicas whose slots no longer exist, e.g. because\n    // they were dropped outside of zero-cache's control (such as in an\n    // upstream failover). Slot names are reused, so such rows would otherwise\n    // be associated with a new slot of the same name. This is done (and\n    // committed) before the slot is created, both so that the rows are never\n    // associated with it, and because CREATE_REPLICATION_SLOT waits for\n    // transactions that have written (i.e. been assigned an xid).\n    const orphaned = await tx<{id: string}[]> /*sql*/ `\n      DELETE FROM ${tx(replicasTable)} replica\n        WHERE NOT EXISTS (\n          SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n        )\n        RETURNING id;\n    `;\n    if (orphaned.length) {\n      lc.warn?.(`deleted replicas whose slots no longer exist`, {\n        replicas: orphaned.map(({id}) => id),\n      });\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAwFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB,qCAAqC;;AAGpE,IAAa,2BAAb,cAA8C,MAAM;CAClD,OAAgB;AAClB;AAKA,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,GACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAChE,MAAM,aAAa,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAAS;CAEtF,IAAI;EACF,MAAM,QAAQ,MAAM,UAClB,YACA,kCACF;EACA,IAAI,UAAU,aACZ,MAAM,IAAI,yBACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;EAEF,MAAM,CAAC,QAAQ;EACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;EACtD,OAAO;CACT,SAAS,GAAG;EACV,IACE,kBACC,gBAAgB,GAAG,qBAAqB,KACvC,aAAa,2BAKf,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAEzE,MAAM;CACR;AACF;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;AASA,SAAS,sBAAsB,GAAqB;CAClD,OACE,gBAAgB,GAAG,qBAAqB,KACxC,aAAa,4BACZ,aAAa,SAAU,EAAuB,SAAS;AAE5D;AAKA,IAAM,uCAAuC,OAAU;AAIvD,IAAM,uCAAuC;;;;;;AAc7C,SAAS,qBAAqB,IAAgB,iBAAyB;CACrE,OAAO,SACL,KACE;EACE,KAAK;GACJ,iBAAiB;GACjB,iBAAiB;EAClB,YAAY,GAAE,qBAAqB,gBAAe;CACpD,GAEA,GAAG,OACL,CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACA,EACE,aACA,eAAe,sCACf,eAAe,yCACgB,CAAC,GACC;CACnC,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAClD,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI,uBAAuB;CAC3B,IAAI,wBAAwB;CAE5B,KAAK,IAAI,UAAU,IAAK,WAAW;EACjC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAKvC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;EACvE,MAAM,cAAc,qBAAqB,KAAK,GAAG,YAAY,MAAM;EAInE,MAAM,oBACJ,QAAQ,WAAW,CACjB,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC,GACnC,YAAY,IAAI,EAAC,SAAS,EAAC,CAAC,CAC9B,CAAC;EAEH,IAAI;EACJ,IAAI;EACJ,IAAI,eAAe;EACnB,IAAI;GACF,MAAM,WAAW,oCAAoC,SAAS;GAE9D,IAAI,UAAA,GAAuB;IAIzB,MAAM,SAAS,MAAM,qBAAqB,KAAK,OAAO,KAAK;IAC3D,GAAG,OAAO,6BAA6B,SAAS,EAAC,UAAU,OAAM,CAAC;IAClE,IAAI,OAAO,QAAQ;KACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;KAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;IAEJ;GACF;GAGA,MAAM,QAAQ,MAAM,GAA8B;;iCAEvB,iBAAiB,IAAI;QAC9C,OAAO;GACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;GAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;IAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;IAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;KAC7B,WAAW;KACX;IACF;GACF;GAEA,eAAe;GACf,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;IAC/D;IACA;IACA;IACA,eAAe;GACjB,CAAC;GACD,eAAe;GACf,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;GACjE,MAAM,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC;GACzC,iBAAiB,MAAM,6BAA6B,IAAI;IACtD,IAAI,wBAAwB,GAAG;IAC/B,MAAM,KAAK;IACX,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,KAAK,gBAAgB,CAAC;GAC7C,CAAC;GAED,MAAM,UAAU,MAAM,MAAM,KAAK,OAAM,OAAM;IAC3C,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IACA,OAAO,KACL,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBACvB;GACF,CAAC;GAED,OAAO;IAAC;IAAM;IAAkB;IAAgB;GAAO;EACzD,SAAS,GAAG;GAIV,MAAM,YAAY;GAClB,IACE,CAAC,wBACD,gBAAgB,GAAG,yBAAyB,GAC5C;IAKA,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD,uBAAuB;IACvB;GACF;GAIA,IACE,CAAC,yBACD,gBAAgB,GAAG,+BAA+B,GAClD;IACA,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC,wBAAwB;IACxB;GACF;GACA,IAAI,gBACF,IAAI;IACF,MAAM,WAAW,cAAc;GACjC,QAAQ,CAAC;GAMX,MAAM,YAAY,KAAK,IAAI,IAAI;GAC/B,IACE,gBACA,sBAAsB,CAAC,KACvB,YAAY,cACZ;IACA,GAAG,OACD,oBAAoB,SAAS,sDACf,QAAQ,IAAI,UAAU,kBACpC,CACF;IACA,MAAM,MAAM,gBAAgB,IAAI,IAAI,KAAK,OAAO,EAAE;IAClD;GACF;GACA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR,UAAU;GACR,MAAM,YAAY;EACpB;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,cACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;CACJ,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAE9E,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aACvB;GAEA,cAAc,MAAM,6BAA6B,IAAI;IACnD,IAAI,wBAAwB,GAAG;IAC/B;IACA,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,GAAG,CAAC;GAC3B,CAAC;GACD,OAAO;IACL;IACA;IACA;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aACF,IAAI;GACF,MAAM,WAAW,WAAW;EAC9B,QAAQ,CAAC;EAEX,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,EAC9B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -13,7 +13,7 @@ import { discoverChangeStreamerAddress } from "./schema/tables.js";
 import "./change-streamer.js";
 import "./snapshot-message.js";
 import { subscribeDownstreamSchema, subscribeUpstreamSchema } from "./subscribe.js";
-import WebSocket from "ws";
+import WebSocket$1 from "ws";
 import websocket from "@fastify/websocket";
 //#region ../zero-cache/src/services/change-streamer/change-streamer-http.ts
 var MIN_SUPPORTED_PROTOCOL_VERSION = 4;
@@ -210,7 +210,7 @@ var ChangeStreamerHttpClient = class {
 	* replication-manager is the one that serves the subscription.
 	*/
 	async connect(taskID) {
-		const ws = new WebSocket(await this.#resolveChangeStreamer(SUBSCRIBE_PATH) + `?${new URLSearchParams({ taskID }).toString()}`, { handshakeTimeout: CONNECT_TIMEOUT_MS });
+		const ws = new WebSocket$1(await this.#resolveChangeStreamer(SUBSCRIBE_PATH) + `?${new URLSearchParams({ taskID }).toString()}`, { handshakeTimeout: CONNECT_TIMEOUT_MS });
 		const outbound = Subscription.create();
 		return {
 			instream: await streamInternalWithSize(this.#lc, ws, subscribeDownstreamSchema, outbound),

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#changes","#reserveSnapshot","#subscribe","#receiveWebsocket","#ensureChangeStreamerStarted","#forwardReservation","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  readHead,\n  streamInternal,\n  streamInternalStringified,\n  streamInternalWithSize,\n  type Sized,\n  type Source,\n} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport {\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {type SnapshotMessage} from './snapshot-message.ts';\nimport {\n  subscribeDownstreamSchema,\n  subscribeUpstreamSchema,\n  type ReservedMessage,\n  type SnapshotStatus,\n  type SubscribeContext,\n  type SubscribeDownstream,\n  type SubscribeUpstream,\n} from './subscribe.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\n// The merged `/subscribe` protocol was introduced at v7.\nconst MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst SUBSCRIBE_PATH_PATTERN = '/replication/:version/subscribe';\nconst PATH_REGEX =\n  /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot|subscribe)$/;\n\nconst SUBSCRIBE_PATH = `/replication/v${PROTOCOL_VERSION}/subscribe`;\n\n// Max time for a client to establish the /subscribe websocket (TCP connect\n// plus upgrade) before abandoning the attempt.\nconst CONNECT_TIMEOUT_MS = 10_000;\n\ntype Options = HttpOptions & {\n  startupDelayMs: number;\n  config?:\n    | Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'>\n    | undefined;\n  getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#changes);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n      fastify.get(SUBSCRIBE_PATH_PATTERN, {websocket: true}, this.#subscribe);\n\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(\n          lc,\n          opts.config ?? {\n            adminPassword: undefined,\n            operatorPassword: undefined,\n          },\n          req,\n          res,\n          opts.getProfileWorker,\n          undefined,\n          'change-streamer',\n        ),\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes' | 'subscribe'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot' | 'subscribe',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#changes(ws, msg);\n      case 'subscribe':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      // Send-only for now: the reservation carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternal(this._lc, ws, undefined, downstream);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #changes = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      // Send-only for now: the subscription carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternalStringified(this._lc, ws, undefined, downstream, {\n        batched: ctx.wsBatched,\n      });\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  /**\n   * The merged v7 `/subscribe` handler. A single bidirectional connection\n   * carries, in sequence: an optional reservation phase (`reserve-snapshot` →\n   * `reserved`) and then the subscription phase (`start-subscription` → change\n   * stream). This ensures that the subscriber subscribes to the same task from\n   * which it restored the backup.\n   */\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    let reservation: Source<SnapshotMessage> | undefined;\n    const outbound = Subscription.create<string | PreSerializedBatch>();\n\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      const protocolVersion = checkProtocolVersion(url.pathname);\n      if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) {\n        throw new Error(\n          `the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} ` +\n            `(client is at v${protocolVersion})`,\n        );\n      }\n\n      const instream = await streamInternalStringified(\n        this._lc,\n        ws,\n        subscribeUpstreamSchema,\n        outbound,\n        {batched: true},\n      );\n\n      // Keep reading upstream control messages for the life of the connection.\n      // Do not manually exit loop, as that would close the socket. The loop\n      // exits naturally when the connection closes.\n      for await (const msg of instream) {\n        if (msg[0] === 'reserve-snapshot') {\n          const {taskID} = msg[1];\n          this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n          reservation =\n            await this.#changeStreamer.startSnapshotReservation(taskID);\n          void this.#forwardReservation(reservation, outbound);\n        } else if (msg[0] === 'start-subscription') {\n          const ctx: SubscriberContext = {\n            ...msg[1],\n            protocolVersion,\n            wsBatched: true,\n          };\n          if (ctx.mode === 'serving') {\n            this.#ensureChangeStreamerStarted('incoming subscription');\n          }\n          // subscribe() registers the subscriber and then releases any\n          // reservation for that task.\n          reservation = undefined;\n          await this.#changeStreamer.subscribe(ctx, outbound);\n        }\n      }\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    } finally {\n      // Covers a reservation-phase disconnect (instream ended before a\n      // subscription started); idempotent with the close handler above.\n      reservation?.cancel();\n    }\n  };\n\n  /**\n   * Forwards a snapshot reservation's single `['status', ...]` message onto\n   * the merged connection's `outbound` sink as a `['reserved', ...]` message.\n   * The loop then blocks on the held-open reservation until it is cancelled,\n   * by `subscribe()` at the subscription handoff, or by the reservation-phase\n   * teardown on disconnect.\n   */\n  async #forwardReservation(\n    reservation: Source<SnapshotMessage>,\n    outbound: Subscription<string | PreSerializedBatch>,\n  ) {\n    try {\n      for await (const [, status] of reservation) {\n        // Translate the /snapshot protocol's ['status', {tag: 'status', ...}]\n        // message to the /subscribe protocol's ['reserved', {tag: 'snapshot', ...}].\n        outbound.push(\n          BigIntJSON.stringify([\n            'reserved',\n            {...status, tag: 'snapshot'},\n          ] satisfies ReservedMessage),\n        );\n      }\n    } catch (e) {\n      this._lc.warn?.(`error forwarding snapshot reservation`, e);\n      outbound.fail(e instanceof Error ? e : new Error(String(e)));\n    }\n  }\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason?: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(\n        `starting ChangeStreamerService ${reason ? `(${reason})` : ''}`,\n      );\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n    if (startupDelayMs === 0) {\n      // In RMv2, there is no need to delay starting the change streamer\n      // because RM startup is non-disruptive.\n      this.#ensureChangeStreamerStarted();\n    } else {\n      // In RMv1, starting the change-streamer forcibly shuts down the\n      // previous change-streamer, causing view-syncers to reconnect.\n      // If this replication-manager has just started, the routing layer may\n      // not have registered it with DNS, as that only happens after it\n      // confirms health checks. To minimize downtime, the takeover is\n      // delayed for the configured startupDelayMs _after_ beginning to\n      // advertise readiness.\n      void readinessGate.then(() => {\n        this.#lc.info?.(\n          `waiting ${startupDelayMs}ms before taking over the change log`,\n        );\n        this._state.setTimeout(\n          () =>\n            this.#ensureChangeStreamerStarted(\n              `startup delay elapsed (${startupDelayMs} ms)`,\n            ),\n          startupDelayMs,\n        );\n      });\n    }\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport interface SnapshotReserver {\n  reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }>;\n}\n\nexport interface SubscriptionStarter {\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\nexport interface ReservationFollowup extends SubscriptionStarter {\n  /**\n   * A signal to indicate that the followup is no longer valid (i.e. the\n   * connection to the reserving task has been severed).\n   */\n  readonly signal: AbortSignal;\n\n  /**\n   * Abandons the reservation without subscribing, closing the connection that\n   * was held open to pin the change-log floor. Used when the caller cannot\n   * proceed to `subscribe()` (e.g. process shutdown while a restore is in\n   * progress) and must not leave the reservation's connection open forever.\n   */\n  cancel(reason?: Error): void;\n}\n\nexport class ChangeStreamerHttpClient\n  implements ChangeStreamer, SnapshotReserver, SubscriptionStarter\n{\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }> {\n    const {instream, outbound} = await this.connect(taskID);\n    outbound.push(['reserve-snapshot', {taskID}]);\n\n    const {head, rest} = readHead(instream);\n    const item = await head.next();\n    if (!item.value) {\n      throw new Error('stream closed without snapshot reservation');\n    }\n    const {data: msg} = item.value;\n    if (msg[0] !== 'reserved') {\n      throw new Error(`expected \"reserved\" message but got ${msg[0]}`);\n    }\n    const reserved: SnapshotStatus = msg[1];\n    const followup: ReservationFollowup = {\n      subscribe: ctx => {\n        outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n        return Promise.resolve(rest);\n      },\n      signal: outbound.signal,\n      cancel: reason => outbound.cancel(reason),\n    };\n\n    return {reserved, followup};\n  }\n\n  async subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>> {\n    const {instream, outbound} = await this.connect(ctx.taskID);\n    outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n    return instream;\n  }\n\n  /**\n   * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the\n   * inbound change/reservation stream and an `outbound` sink into which the\n   * caller pushes the `reserve-snapshot` / `start-subscription` control\n   * messages that drive the connection's phases. The single connection spans\n   * the reservation, backup restore, and subscription, so the reserving\n   * replication-manager is the one that serves the subscription.\n   */\n  // exported for testing\n  async connect(taskID: string): Promise<{\n    instream: Source<Sized<SubscribeDownstream>>;\n    outbound: Subscription<SubscribeUpstream>;\n  }> {\n    const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);\n\n    // taskID is carried in the query for observability/routing; the server\n    // reads the authoritative taskID from the control messages themselves.\n    const params = new URLSearchParams({taskID});\n    // Bound the TCP connect + HTTP upgrade. Without this, a SYN black-holed\n    // by a stale Service endpoint (e.g. a just-terminated replication-manager\n    // pod) is retransmitted by the kernel for ~2 minutes before ETIMEDOUT.\n    // Failing fast lets the caller's retry open a fresh connection that is\n    // routed to a live endpoint.\n    const ws = new WebSocket(uri + `?${params.toString()}`, {\n      handshakeTimeout: CONNECT_TIMEOUT_MS,\n    });\n\n    const outbound = Subscription.create<SubscribeUpstream>();\n    const instream = await streamInternalWithSize(\n      this.#lc,\n      ws,\n      subscribeDownstreamSchema,\n      outbound,\n    );\n    return {instream, outbound};\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    wsBatched: params.getBoolean('wsBatched'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// Projects a client-side SubscriberContext onto the start-subscription\n// message payload, dropping the fields that are implicit on the merged\n// connection: protocolVersion (from the request path), wsBatched (always on).\nfunction toStartSubscriptionContext(ctx: SubscriberContext): SubscribeContext {\n  const {taskID, id, mode, replicaVersion, watermark, backfills} = ctx;\n  return {taskID, id, mode, replicaVersion, watermark, backfills};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,IAAM,iCAAiC;AAGvC,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAC/B,IAAM,aACJ;AAEF,IAAM,iBAAiB;AAIvB,IAAM,qBAAqB;AAU3B,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,QAAQ;GAClE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GACA,QAAQ,IAAI,wBAAwB,EAAC,WAAW,KAAI,GAAG,KAAKC,UAAU;GAEtE,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU;IACb,eAAe,KAAA;IACf,kBAAkB,KAAA;GACpB,GACA,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,SAAS,IAAI,GAAG;GAC9B,KAAK,aACH,OAAO,KAAKE,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKE,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;GAG5D,eAAoB,KAAK,KAAK,IAAI,KAAA,GAAW,UAAU;EACzD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,WAAoB,OAAO,IAAe,QAAwB;EAChE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKK,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKL,gBAAgB,UAAU,GAAG;GAG3D,0BAA+B,KAAK,KAAK,IAAI,KAAA,GAAW,YAAY,EAClE,SAAS,IAAI,UACf,CAAC;EACH,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;;;;;;;;CASA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;EACJ,MAAM,WAAW,aAAa,OAAoC;EAElE,IAAI;GAKF,MAAM,kBAAkB,qBAAqB,IAJ7B,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBAEqB,EAAI,QAAQ;GACzD,IAAI,kBAAkB,gCACpB,MAAM,IAAI,MACR,8CAA8C,+BAA+B,kBACzD,gBAAgB,EACtC;GAGF,MAAM,WAAW,MAAM,0BACrB,KAAK,KACL,IACA,yBACA,UACA,EAAC,SAAS,KAAI,CAChB;GAKA,WAAW,MAAM,OAAO,UACtB,IAAI,IAAI,OAAO,oBAAoB;IACjC,MAAM,EAAC,WAAU,IAAI;IACrB,KAAKK,6BAA6B,+BAA+B;IACjE,cACE,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;IAC5D,KAAUM,oBAAoB,aAAa,QAAQ;GACrD,OAAO,IAAI,IAAI,OAAO,sBAAsB;IAC1C,MAAM,MAAyB;KAC7B,GAAG,IAAI;KACP;KACA,WAAW;IACb;IACA,IAAI,IAAI,SAAS,WACf,KAAKD,6BAA6B,uBAAuB;IAI3D,cAAc,KAAA;IACd,MAAM,KAAKL,gBAAgB,UAAU,KAAK,QAAQ;GACpD;EAEJ,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD,UAAU;GAGR,aAAa,OAAO;EACtB;CACF;;;;;;;;CASA,MAAMM,oBACJ,aACA,UACA;EACA,IAAI;GACF,WAAW,MAAM,GAAG,WAAW,aAG7B,SAAS,KACP,WAAW,UAAU,CACnB,YACA;IAAC,GAAG;IAAQ,KAAK;GAAU,CAC7B,CAA2B,CAC7B;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,SAAS,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EAC7D;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAiB;EAC5C,IAAI,CAAC,KAAKC,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OACP,kCAAkC,SAAS,IAAI,OAAO,KAAK,IAC7D;GACA,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKR;EAC3D,IAAI,mBAAmB,GAGrB,KAAKM,6BAA6B;OASlC,cAAmB,WAAW;GAC5B,KAAKP,IAAI,OACP,WAAW,eAAe,qCAC5B;GACA,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;EACF,CAAC;CAEL;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKE,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AA+BA,IAAa,2BAAb,MAEA;CACE;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAGnB;EACD,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,MAAM;EACtD,SAAS,KAAK,CAAC,oBAAoB,EAAC,OAAM,CAAC,CAAC;EAE5C,MAAM,EAAC,MAAM,SAAQ,SAAS,QAAQ;EACtC,MAAM,OAAO,MAAM,KAAK,KAAK;EAC7B,IAAI,CAAC,KAAK,OACR,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,EAAC,MAAM,QAAO,KAAK;EACzB,IAAI,IAAI,OAAO,YACb,MAAM,IAAI,MAAM,uCAAuC,IAAI,IAAI;EAYjE,OAAO;GAAC,UAVyB,IAAI;GAUnB,UAAA;IARhB,YAAW,QAAO;KAChB,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;KACrE,OAAO,QAAQ,QAAQ,IAAI;IAC7B;IACA,QAAQ,SAAS;IACjB,SAAQ,WAAU,SAAS,OAAO,MAAM;GAGxB;EAAQ;CAC5B;CAEA,MAAM,UACJ,KAC6C;EAC7C,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,IAAI,MAAM;EAC1D,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;EACrE,OAAO;CACT;;;;;;;;;CAWA,MAAM,QAAQ,QAGX;EAWD,MAAM,KAAK,IAAI,UAAU,MAVP,KAAKa,uBAAuB,cAAc,IAU7B,IAAI,IANhB,gBAAgB,EAAC,OAAM,CAMP,EAAO,SAAS,KAAK,EACtD,kBAAkB,mBACpB,CAAC;EAED,MAAM,WAAW,aAAa,OAA0B;EAOxD,OAAO;GAAC,UAAA,MANe,uBACrB,KAAKb,KACL,IACA,2BACA,QACF;GACkB;EAAQ;CAC5B;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EAIvC,WAAW,OAAO,WAAW,WAAW;CAC1C;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAKA,SAAS,2BAA2B,KAA0C;CAC5E,MAAM,EAAC,QAAQ,IAAI,MAAM,gBAAgB,WAAW,cAAa;CACjE,OAAO;EAAC;EAAQ;EAAI;EAAM;EAAgB;EAAW;CAAS;AAChE"}
+{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#changes","#reserveSnapshot","#subscribe","#receiveWebsocket","#ensureChangeStreamerStarted","#forwardReservation","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  readHead,\n  streamInternal,\n  streamInternalStringified,\n  streamInternalWithSize,\n  type Sized,\n  type Source,\n} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport {\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {type SnapshotMessage} from './snapshot-message.ts';\nimport {\n  subscribeDownstreamSchema,\n  subscribeUpstreamSchema,\n  type ReservedMessage,\n  type SnapshotStatus,\n  type SubscribeContext,\n  type SubscribeDownstream,\n  type SubscribeUpstream,\n} from './subscribe.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\n// The merged `/subscribe` protocol was introduced at v7.\nconst MIN_SUBSCRIBE_PROTOCOL_VERSION = 7;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst SUBSCRIBE_PATH_PATTERN = '/replication/:version/subscribe';\nconst PATH_REGEX =\n  /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot|subscribe)$/;\n\nconst SUBSCRIBE_PATH = `/replication/v${PROTOCOL_VERSION}/subscribe`;\n\n// Max time for a client to establish the /subscribe websocket (TCP connect\n// plus upgrade) before abandoning the attempt.\nconst CONNECT_TIMEOUT_MS = 10_000;\n\ntype Options = HttpOptions & {\n  startupDelayMs: number;\n  config?:\n    | Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'>\n    | undefined;\n  getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#changes);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n      fastify.get(SUBSCRIBE_PATH_PATTERN, {websocket: true}, this.#subscribe);\n\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(\n          lc,\n          opts.config ?? {\n            adminPassword: undefined,\n            operatorPassword: undefined,\n          },\n          req,\n          res,\n          opts.getProfileWorker,\n          undefined,\n          'change-streamer',\n        ),\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes' | 'subscribe'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot' | 'subscribe',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#changes(ws, msg);\n      case 'subscribe':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      // Send-only for now: the reservation carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternal(this._lc, ws, undefined, downstream);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #changes = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      // Send-only for now: the subscription carries no inbound application\n      // messages (only transport acks, demuxed internally).\n      void streamInternalStringified(this._lc, ws, undefined, downstream, {\n        batched: ctx.wsBatched,\n      });\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  /**\n   * The merged v7 `/subscribe` handler. A single bidirectional connection\n   * carries, in sequence: an optional reservation phase (`reserve-snapshot` →\n   * `reserved`) and then the subscription phase (`start-subscription` → change\n   * stream). This ensures that the subscriber subscribes to the same task from\n   * which it restored the backup.\n   */\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    let reservation: Source<SnapshotMessage> | undefined;\n    const outbound = Subscription.create<string | PreSerializedBatch>();\n\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      const protocolVersion = checkProtocolVersion(url.pathname);\n      if (protocolVersion < MIN_SUBSCRIBE_PROTOCOL_VERSION) {\n        throw new Error(\n          `the /subscribe endpoint requires protocol v${MIN_SUBSCRIBE_PROTOCOL_VERSION} ` +\n            `(client is at v${protocolVersion})`,\n        );\n      }\n\n      const instream = await streamInternalStringified(\n        this._lc,\n        ws,\n        subscribeUpstreamSchema,\n        outbound,\n        {batched: true},\n      );\n\n      // Keep reading upstream control messages for the life of the connection.\n      // Do not manually exit loop, as that would close the socket. The loop\n      // exits naturally when the connection closes.\n      for await (const msg of instream) {\n        if (msg[0] === 'reserve-snapshot') {\n          const {taskID} = msg[1];\n          this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n          reservation =\n            await this.#changeStreamer.startSnapshotReservation(taskID);\n          void this.#forwardReservation(reservation, outbound);\n        } else if (msg[0] === 'start-subscription') {\n          const ctx: SubscriberContext = {\n            ...msg[1],\n            protocolVersion,\n            wsBatched: true,\n          };\n          if (ctx.mode === 'serving') {\n            this.#ensureChangeStreamerStarted('incoming subscription');\n          }\n          // subscribe() registers the subscriber and then releases any\n          // reservation for that task.\n          reservation = undefined;\n          await this.#changeStreamer.subscribe(ctx, outbound);\n        }\n      }\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    } finally {\n      // Covers a reservation-phase disconnect (instream ended before a\n      // subscription started); idempotent with the close handler above.\n      reservation?.cancel();\n    }\n  };\n\n  /**\n   * Forwards a snapshot reservation's single `['status', ...]` message onto\n   * the merged connection's `outbound` sink as a `['reserved', ...]` message.\n   * The loop then blocks on the held-open reservation until it is cancelled,\n   * by `subscribe()` at the subscription handoff, or by the reservation-phase\n   * teardown on disconnect.\n   */\n  async #forwardReservation(\n    reservation: Source<SnapshotMessage>,\n    outbound: Subscription<string | PreSerializedBatch>,\n  ) {\n    try {\n      for await (const [, status] of reservation) {\n        // Translate the /snapshot protocol's ['status', {tag: 'status', ...}]\n        // message to the /subscribe protocol's ['reserved', {tag: 'snapshot', ...}].\n        outbound.push(\n          BigIntJSON.stringify([\n            'reserved',\n            {...status, tag: 'snapshot'},\n          ] satisfies ReservedMessage),\n        );\n      }\n    } catch (e) {\n      this._lc.warn?.(`error forwarding snapshot reservation`, e);\n      outbound.fail(e instanceof Error ? e : new Error(String(e)));\n    }\n  }\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason?: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(\n        `starting ChangeStreamerService ${reason ? `(${reason})` : ''}`,\n      );\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n    if (startupDelayMs === 0) {\n      // In RMv2, there is no need to delay starting the change streamer\n      // because RM startup is non-disruptive.\n      this.#ensureChangeStreamerStarted();\n    } else {\n      // In RMv1, starting the change-streamer forcibly shuts down the\n      // previous change-streamer, causing view-syncers to reconnect.\n      // If this replication-manager has just started, the routing layer may\n      // not have registered it with DNS, as that only happens after it\n      // confirms health checks. To minimize downtime, the takeover is\n      // delayed for the configured startupDelayMs _after_ beginning to\n      // advertise readiness.\n      void readinessGate.then(() => {\n        this.#lc.info?.(\n          `waiting ${startupDelayMs}ms before taking over the change log`,\n        );\n        this._state.setTimeout(\n          () =>\n            this.#ensureChangeStreamerStarted(\n              `startup delay elapsed (${startupDelayMs} ms)`,\n            ),\n          startupDelayMs,\n        );\n      });\n    }\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport interface SnapshotReserver {\n  reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }>;\n}\n\nexport interface SubscriptionStarter {\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\nexport interface ReservationFollowup extends SubscriptionStarter {\n  /**\n   * A signal to indicate that the followup is no longer valid (i.e. the\n   * connection to the reserving task has been severed).\n   */\n  readonly signal: AbortSignal;\n\n  /**\n   * Abandons the reservation without subscribing, closing the connection that\n   * was held open to pin the change-log floor. Used when the caller cannot\n   * proceed to `subscribe()` (e.g. process shutdown while a restore is in\n   * progress) and must not leave the reservation's connection open forever.\n   */\n  cancel(reason?: Error): void;\n}\n\nexport class ChangeStreamerHttpClient\n  implements ChangeStreamer, SnapshotReserver, SubscriptionStarter\n{\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<{\n    reserved: SnapshotStatus;\n    followup: ReservationFollowup;\n  }> {\n    const {instream, outbound} = await this.connect(taskID);\n    outbound.push(['reserve-snapshot', {taskID}]);\n\n    const {head, rest} = readHead(instream);\n    const item = await head.next();\n    if (!item.value) {\n      throw new Error('stream closed without snapshot reservation');\n    }\n    const {data: msg} = item.value;\n    if (msg[0] !== 'reserved') {\n      throw new Error(`expected \"reserved\" message but got ${msg[0]}`);\n    }\n    const reserved: SnapshotStatus = msg[1];\n    const followup: ReservationFollowup = {\n      subscribe: ctx => {\n        outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n        return Promise.resolve(rest);\n      },\n      signal: outbound.signal,\n      cancel: reason => outbound.cancel(reason),\n    };\n\n    return {reserved, followup};\n  }\n\n  async subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>> {\n    const {instream, outbound} = await this.connect(ctx.taskID);\n    outbound.push(['start-subscription', toStartSubscriptionContext(ctx)]);\n    return instream;\n  }\n\n  /**\n   * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the\n   * inbound change/reservation stream and an `outbound` sink into which the\n   * caller pushes the `reserve-snapshot` / `start-subscription` control\n   * messages that drive the connection's phases. The single connection spans\n   * the reservation, backup restore, and subscription, so the reserving\n   * replication-manager is the one that serves the subscription.\n   */\n  // exported for testing\n  async connect(taskID: string): Promise<{\n    instream: Source<Sized<SubscribeDownstream>>;\n    outbound: Subscription<SubscribeUpstream>;\n  }> {\n    const uri = await this.#resolveChangeStreamer(SUBSCRIBE_PATH);\n\n    // taskID is carried in the query for observability/routing; the server\n    // reads the authoritative taskID from the control messages themselves.\n    const params = new URLSearchParams({taskID});\n    // Bound the TCP connect + HTTP upgrade. Without this, a SYN black-holed\n    // by a stale Service endpoint (e.g. a just-terminated replication-manager\n    // pod) is retransmitted by the kernel for ~2 minutes before ETIMEDOUT.\n    // Failing fast lets the caller's retry open a fresh connection that is\n    // routed to a live endpoint.\n    const ws = new WebSocket(uri + `?${params.toString()}`, {\n      handshakeTimeout: CONNECT_TIMEOUT_MS,\n    });\n\n    const outbound = Subscription.create<SubscribeUpstream>();\n    const instream = await streamInternalWithSize(\n      this.#lc,\n      ws,\n      subscribeDownstreamSchema,\n      outbound,\n    );\n    return {instream, outbound};\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    wsBatched: params.getBoolean('wsBatched'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// Projects a client-side SubscriberContext onto the start-subscription\n// message payload, dropping the fields that are implicit on the merged\n// connection: protocolVersion (from the request path), wsBatched (always on).\nfunction toStartSubscriptionContext(ctx: SubscriberContext): SubscribeContext {\n  const {taskID, id, mode, replicaVersion, watermark, backfills} = ctx;\n  return {taskID, id, mode, replicaVersion, watermark, backfills};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA4CA,IAAM,iCAAiC;AAGvC,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAC/B,IAAM,aACJ;AAEF,IAAM,iBAAiB;AAIvB,IAAM,qBAAqB;AAU3B,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,QAAQ;GAClE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GACA,QAAQ,IAAI,wBAAwB,EAAC,WAAW,KAAI,GAAG,KAAKC,UAAU;GAEtE,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU;IACb,eAAe,KAAA;IACf,kBAAkB,KAAA;GACpB,GACA,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,SAAS,IAAI,GAAG;GAC9B,KAAK,aACH,OAAO,KAAKE,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKE,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;GAG5D,eAAoB,KAAK,KAAK,IAAI,KAAA,GAAW,UAAU;EACzD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,WAAoB,OAAO,IAAe,QAAwB;EAChE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKK,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKL,gBAAgB,UAAU,GAAG;GAG3D,0BAA+B,KAAK,KAAK,IAAI,KAAA,GAAW,YAAY,EAClE,SAAS,IAAI,UACf,CAAC;EACH,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;;;;;;;;CASA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;EACJ,MAAM,WAAW,aAAa,OAAoC;EAElE,IAAI;GAKF,MAAM,kBAAkB,qBAAqB,IAJ7B,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBAEqB,EAAI,QAAQ;GACzD,IAAI,kBAAkB,gCACpB,MAAM,IAAI,MACR,8CAA8C,+BAA+B,kBACzD,gBAAgB,EACtC;GAGF,MAAM,WAAW,MAAM,0BACrB,KAAK,KACL,IACA,yBACA,UACA,EAAC,SAAS,KAAI,CAChB;GAKA,WAAW,MAAM,OAAO,UACtB,IAAI,IAAI,OAAO,oBAAoB;IACjC,MAAM,EAAC,WAAU,IAAI;IACrB,KAAKK,6BAA6B,+BAA+B;IACjE,cACE,MAAM,KAAKL,gBAAgB,yBAAyB,MAAM;IAC5D,KAAUM,oBAAoB,aAAa,QAAQ;GACrD,OAAO,IAAI,IAAI,OAAO,sBAAsB;IAC1C,MAAM,MAAyB;KAC7B,GAAG,IAAI;KACP;KACA,WAAW;IACb;IACA,IAAI,IAAI,SAAS,WACf,KAAKD,6BAA6B,uBAAuB;IAI3D,cAAc,KAAA;IACd,MAAM,KAAKL,gBAAgB,UAAU,KAAK,QAAQ;GACpD;EAEJ,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD,UAAU;GAGR,aAAa,OAAO;EACtB;CACF;;;;;;;;CASA,MAAMM,oBACJ,aACA,UACA;EACA,IAAI;GACF,WAAW,MAAM,GAAG,WAAW,aAG7B,SAAS,KACP,WAAW,UAAU,CACnB,YACA;IAAC,GAAG;IAAQ,KAAK;GAAU,CAC7B,CAA2B,CAC7B;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,SAAS,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EAC7D;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAiB;EAC5C,IAAI,CAAC,KAAKC,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OACP,kCAAkC,SAAS,IAAI,OAAO,KAAK,IAC7D;GACA,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKR;EAC3D,IAAI,mBAAmB,GAGrB,KAAKM,6BAA6B;OASlC,cAAmB,WAAW;GAC5B,KAAKP,IAAI,OACP,WAAW,eAAe,qCAC5B;GACA,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;EACF,CAAC;CAEL;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKE,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AA+BA,IAAa,2BAAb,MAEA;CACE;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAGnB;EACD,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,MAAM;EACtD,SAAS,KAAK,CAAC,oBAAoB,EAAC,OAAM,CAAC,CAAC;EAE5C,MAAM,EAAC,MAAM,SAAQ,SAAS,QAAQ;EACtC,MAAM,OAAO,MAAM,KAAK,KAAK;EAC7B,IAAI,CAAC,KAAK,OACR,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,EAAC,MAAM,QAAO,KAAK;EACzB,IAAI,IAAI,OAAO,YACb,MAAM,IAAI,MAAM,uCAAuC,IAAI,IAAI;EAYjE,OAAO;GAAC,UAVyB,IAAI;GAUnB,UAAA;IARhB,YAAW,QAAO;KAChB,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;KACrE,OAAO,QAAQ,QAAQ,IAAI;IAC7B;IACA,QAAQ,SAAS;IACjB,SAAQ,WAAU,SAAS,OAAO,MAAM;GAGxB;EAAQ;CAC5B;CAEA,MAAM,UACJ,KAC6C;EAC7C,MAAM,EAAC,UAAU,aAAY,MAAM,KAAK,QAAQ,IAAI,MAAM;EAC1D,SAAS,KAAK,CAAC,sBAAsB,2BAA2B,GAAG,CAAC,CAAC;EACrE,OAAO;CACT;;;;;;;;;CAWA,MAAM,QAAQ,QAGX;EAWD,MAAM,KAAK,IAAI,YAAU,MAVP,KAAKa,uBAAuB,cAAc,IAU7B,IAAI,IANhB,gBAAgB,EAAC,OAAM,CAMP,EAAO,SAAS,KAAK,EACtD,kBAAkB,mBACpB,CAAC;EAED,MAAM,WAAW,aAAa,OAA0B;EAOxD,OAAO;GAAC,UAAA,MANe,uBACrB,KAAKb,KACL,IACA,2BACA,QACF;GACkB;EAAQ;CAC5B;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EAIvC,WAAW,OAAO,WAAW,WAAW;CAC1C;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAKA,SAAS,2BAA2B,KAA0C;CAC5E,MAAM,EAAC,QAAQ,IAAI,MAAM,gBAAgB,WAAW,cAAa;CACjE,OAAO;EAAC;EAAQ;EAAI;EAAM;EAAgB;EAAW;CAAS;AAChE"}