@rocicorp/zero 1.11.0-canary.3 → 1.11.0-canary.30

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js

@@ -1,7 +1,8 @@
 import { sleep } from "../../../../../../shared/src/sleep.js";
+import { getOrCreateHistogram } from "../../../../observability/metrics.js";
 import { id, lit } from "../../../../types/sql.js";
 import "../../../../types/pg.js";
-import { fromBigInt } from "../lsn.js";
+import { fromBigInt, toBigInt } from "../lsn.js";
 import { Subscription } from "../../../../types/subscription.js";
 import { pipe } from "../../../../types/streams.js";
 import { AutoResetSignal } from "../../../change-streamer/schema/tables.js";
@@ -31,6 +32,7 @@ async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSl
         WHERE name = 'wal_sender_timeout'`.simple();
 	const { enabled: livenessEnabled, manualKeepaliveTimeout, inboundTimeoutMs, timerIntervalMs } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);
 	lc.info?.(livenessEnabled ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. Inbound timeout: ${inboundTimeoutMs}ms` + (inboundTimeoutMs === walSenderTimeoutMs * 2 ? "" : " (configured override)") : "wal_sender_timeout is disabled (0); skipping manual keepalives and inbound liveness detection.");
+	const decode = await InitialDecodeTracker.start(lc, session, slot, lsn);
 	const [readable, writable] = await startReplicationStream(lc, session, slot, publications, lsn, retriesIfReplicationSlotActive + 1);
 	let lastAckTime = Date.now();
 	function sendAck(lsn) {
@@ -71,7 +73,9 @@ async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSl
 		sink: messages,
 		parse: (buffer) => {
 			lastReceivedTime = Date.now();
-			return parseStreamMessage(lc, buffer, parser);
+			const msg = parseStreamMessage(lc, buffer, parser);
+			if (msg) decode.observe(msg, lastReceivedTime);
+			return msg;
 		},
 		bufferMessages: 5
 	});
@@ -80,6 +84,100 @@ async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSl
 		acks: { push: sendAck }
 	};
 }
+var INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S = [
+	.1,
+	.5,
+	1,
+	2,
+	5,
+	10,
+	30,
+	60,
+	120,
+	300,
+	600,
+	1200,
+	2400,
+	3600,
+	7200
+];
+function initialDecodeHistogram() {
+	return getOrCreateHistogram("replication", "stream_initial_decode", {
+		description: "Time from START_REPLICATION until upstream has decoded the WAL between the slot's restart_lsn and the start of the stream",
+		unit: "s",
+		bucketBoundaries: INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S
+	});
+}
+/**
+* Measures how long upstream takes to decode the WAL between the slot's
+* `restart_lsn` and the start of the stream, during which the stream
+* delivers nothing but keepalives. This span can be large, e.g. for a slot
+* that was created (or last acked) long before the requested start LSN.
+*
+* The walsender streams from `max(lsn, confirmed_flush_lsn)`, but must
+* decode from `restart_lsn` to rebuild its transaction state. The decode is
+* considered complete upon the first message at or past the start:
+* * any XLogData message, since only transactions committed after the
+*   start are sent, or
+* * a keepalive at or past the start. Keepalives report the decoding
+*   position, except for the first one of the stream, which reports the
+*   slot's `confirmed_flush_lsn` and is thus ignored if it is at or past the
+*   start.
+*
+* Note that on an idle upstream, a walsender whose start is already
+* confirmed may not send a keepalive until other WAL is written or the
+* periodic keepalive (`wal_sender_timeout / 2`) is due, so the measured
+* time is an upper bound when completed by a keepalive.
+*/
+var InitialDecodeTracker = class InitialDecodeTracker {
+	static async start(lc, session, slot, lsn) {
+		let restartLSN;
+		let confirmedFlushLSN;
+		try {
+			const [row] = await session.unsafe(`SELECT restart_lsn AS "restartLSN",
+                  confirmed_flush_lsn AS "confirmedFlushLSN"
+             FROM pg_replication_slots
+             WHERE slot_name = ${lit(slot)}`).simple();
+			restartLSN = row?.restartLSN ? toBigInt(row.restartLSN) : void 0;
+			confirmedFlushLSN = row?.confirmedFlushLSN ? toBigInt(row.confirmedFlushLSN) : void 0;
+		} catch (e) {
+			lc.warn?.(`unable to read position of replication slot ${slot}`, e);
+		}
+		return new InitialDecodeTracker(lc, lsn, restartLSN, confirmedFlushLSN, Date.now());
+	}
+	#lc;
+	#start;
+	#restartLSN;
+	#confirmedFlushLSN;
+	#startTime;
+	#first = true;
+	#done = false;
+	constructor(lc, lsn, restartLSN, confirmedFlushLSN, startTime) {
+		this.#start = confirmedFlushLSN !== void 0 && confirmedFlushLSN > lsn ? confirmedFlushLSN : lsn;
+		this.#lc = lc.withContext("decodeStart", fromBigInt(this.#start));
+		this.#restartLSN = restartLSN;
+		this.#confirmedFlushLSN = confirmedFlushLSN;
+		this.#startTime = startTime;
+		if (restartLSN !== void 0) this.#lc.info?.(`decoding ${this.#start - restartLSN} bytes of WAL from restart_lsn ${fromBigInt(restartLSN)} to the start of the stream`);
+	}
+	/**
+	* Observes a received message, returning the elapsed decode time (ms) if
+	* this message completed the initial decode.
+	*/
+	observe([lsn, msg], now) {
+		if (this.#done) return;
+		const first = this.#first;
+		this.#first = false;
+		if (msg.tag === "keepalive") {
+			if (lsn < this.#start || first && lsn === this.#confirmedFlushLSN) return;
+		}
+		this.#done = true;
+		const elapsed = now - this.#startTime;
+		initialDecodeHistogram().recordMs(elapsed);
+		this.#lc.info?.(`upstream decoded the WAL to the start of the stream in ${elapsed} ms (${msg.tag} at ${fromBigInt(lsn)}` + (this.#restartLSN === void 0 ? ")" : `, ${this.#start - this.#restartLSN} bytes from restart_lsn)`), { elapsed });
+		return elapsed;
+	}
+};
 /**
 * Derives the timings for the replication liveness timer from the server's
 * `wal_sender_timeout` (in milliseconds).
@@ -175,7 +273,7 @@ async function startReplicationStream(lc, session, slot, publications, lsn, maxA
 		if (e instanceof postgres.PostgresError) {
 			if (e.code === PG_OBJECT_IN_USE) {
 				lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);
-				await sleep(10);
+				await sleep(Math.min(10 * 2 ** i, 1e3));
 				continue;
 			}
 			if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {
@@ -207,6 +305,7 @@ async function startReplicationStream(lc, session, slot, publications, lsn, maxA
 async function keepSlotActiveUntilTakenOver(lc, session, slot, dummyPublication, lsn) {
 	const [readable, writeable] = await startReplicationStream(lc, session, slot, [dummyPublication], lsn, 1);
 	lc.info?.(`keeping ${slot} active until taken over ...`);
+	readable.resume();
 	const keepalive = setInterval(() => writeable.write(makeAck(lsn)), 1e4);
 	readable.on("close", async () => {
 		lc.info?.(`released initial reservation of ${slot}`);

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport function createReplicationSessionFor(\n  db: PostgresDB,\n  applicationName: string,\n) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n  inboundTimeoutOverrideMs?: number | undefined,\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = createReplicationSessionFor(db, applicationName);\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. ` +\n          `Inbound timeout: ${inboundTimeoutMs}ms` +\n          (inboundTimeoutMs === walSenderTimeoutMs * 2\n            ? ''\n            : ' (configured override)')\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> inbound timeout ${inboundTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      return parseStreamMessage(lc, buffer, parser);\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * An `inboundTimeoutOverrideMs` (when positive) replaces the default\n * `2x wal_sender_timeout` inbound threshold. The two timeouts protect\n * different parties with asymmetric false-positive costs: `wal_sender_timeout`\n * is the server reaping dead clients (a false positive there costs a cheap\n * reconnect), while the inbound watchdog tears down the stream and replays the\n * in-flight transaction — expensive, and self-defeating when the wal sender is\n * legitimately silent for long stretches (e.g. decoding through WAL from\n * unpublished tables, or assembling a large transaction that pgoutput only\n * sends at commit). Operators who don't own the server setting (e.g. managed\n * or operator-provisioned Postgres with an aggressive cluster-wide\n * `wal_sender_timeout`) can widen the client-side fuse independently.\n *\n * The override has no effect when `wal_sender_timeout` is disabled (0): the\n * server then sends no periodic keepalives, so inbound silence is normal even\n * on an idle healthy connection and any silence-based watchdog would\n * false-positive.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(\n  walSenderTimeoutMs: number,\n  inboundTimeoutOverrideMs?: number | undefined,\n): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.). A 0 / negative / NaN override\n    // falls back to the default rather than degenerating the threshold.\n    inboundTimeoutMs:\n      inboundTimeoutOverrideMs !== undefined && inboundTimeoutOverrideMs > 0\n        ? inboundTimeoutOverrideMs\n        : walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * This essentially reserves the replication slot for this process by marking\n * it as \"active\" in pg_replication_slots, allowing replication slot cleanup\n * logic to distinguish a new-but-initializing replication slot from one that\n * has been abandoned / orphaned.\n *\n * When initialization (e.g. initial-sync) has finished, the server should take\n * over the replication slot using pg_terminate_backend, which will close the\n * session.\n *\n * Note that caller should not close the supplied `session`, as that will\n * also terminate the dummy replication session.\n */\nexport async function keepSlotActiveUntilTakenOver(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  dummyPublication: string,\n  lsn: bigint,\n) {\n  const [readable, writeable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    [dummyPublication],\n    lsn,\n    1,\n  );\n  lc.info?.(`keeping ${slot} active until taken over ...`);\n  // Send periodic status messages upstream to indicate that the session is\n  // alive, but keep the confirmed_flush_lsn at the initial lsn.\n  const keepalive = setInterval(() => writeable.write(makeAck(lsn)), 10_000);\n\n  readable.on('close', async () => {\n    lc.info?.(`released initial reservation of ${slot}`);\n    clearInterval(keepalive);\n    try {\n      await session.end();\n      lc.info?.(`ended session`);\n    } catch (e) {\n      lc.warn?.(`error ending session`, e);\n    }\n  });\n  readable.once('error', e => {\n    lc.info?.(`closing replication session to ${session.options.host}`, e);\n    readable.destroy();\n  });\n  return readable;\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;AAmBA,IAAM,6CAA6C;AAcnD,SAAgB,4BACd,IACA,iBACA;CACA,OAAO,SACL,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;AACF;AAEA,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAClB,0BAIC;CACD,MAAM,UAAU,4BAA4B,IAAI,eAAe;CAQ/D,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,oBAAoB,wBAAwB;CACvE,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,uBAChD,iBAAiB,OACpC,qBAAqB,qBAAqB,IACvC,KACA,4BACN,gGAEN;CAEA,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,wBAC5C,iBAAiB,uDAE3C;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,OAAO,mBAAmB,IAAI,QAAQ,MAAM;EAC9C;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,SAAgB,uBACd,oBACA,0BAMA;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAMA,kBACE,6BAA6B,KAAA,KAAa,2BAA2B,IACjE,2BACA,qBAAqB;EAE3B,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,SACA,MACA,kBACA,KACA;CACA,MAAM,CAAC,UAAU,aAAa,MAAM,uBAClC,IACA,SACA,MACA,CAAC,gBAAgB,GACjB,KACA,CACF;CACA,GAAG,OAAO,WAAW,KAAK,6BAA6B;CAGvD,MAAM,YAAY,kBAAkB,UAAU,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAM;CAEzE,SAAS,GAAG,SAAS,YAAY;EAC/B,GAAG,OAAO,mCAAmC,MAAM;EACnD,cAAc,SAAS;EACvB,IAAI;GACF,MAAM,QAAQ,IAAI;GAClB,GAAG,OAAO,eAAe;EAC3B,SAAS,GAAG;GACV,GAAG,OAAO,wBAAwB,CAAC;EACrC;CACF,CAAC;CACD,SAAS,KAAK,UAAS,MAAK;EAC1B,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,QAAQ,CAAC;EACrE,SAAS,QAAQ;CACnB,CAAC;CACD,OAAO;AACT;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}
+{"version":3,"file":"stream.js","names":["#lc","#start","#restartLSN","#confirmedFlushLSN","#startTime","#done","#first"],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {getOrCreateHistogram} from '../../../../observability/metrics.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt, toBigInt, type LSN} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport function createReplicationSessionFor(\n  db: PostgresDB,\n  applicationName: string,\n) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n  inboundTimeoutOverrideMs?: number | undefined,\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = createReplicationSessionFor(db, applicationName);\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. ` +\n          `Inbound timeout: ${inboundTimeoutMs}ms` +\n          (inboundTimeoutMs === walSenderTimeoutMs * 2\n            ? ''\n            : ' (configured override)')\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const decode = await InitialDecodeTracker.start(lc, session, slot, lsn);\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> inbound timeout ${inboundTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      const msg = parseStreamMessage(lc, buffer, parser);\n      if (msg) {\n        decode.observe(msg, lastReceivedTime);\n      }\n      return msg;\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\nconst INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S = [\n  0.1, 0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200,\n];\n\nfunction initialDecodeHistogram() {\n  return getOrCreateHistogram('replication', 'stream_initial_decode', {\n    description:\n      'Time from START_REPLICATION until upstream has decoded the WAL ' +\n      \"between the slot's restart_lsn and the start of the stream\",\n    unit: 's',\n    bucketBoundaries: INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n/**\n * Measures how long upstream takes to decode the WAL between the slot's\n * `restart_lsn` and the start of the stream, during which the stream\n * delivers nothing but keepalives. This span can be large, e.g. for a slot\n * that was created (or last acked) long before the requested start LSN.\n *\n * The walsender streams from `max(lsn, confirmed_flush_lsn)`, but must\n * decode from `restart_lsn` to rebuild its transaction state. The decode is\n * considered complete upon the first message at or past the start:\n * * any XLogData message, since only transactions committed after the\n *   start are sent, or\n * * a keepalive at or past the start. Keepalives report the decoding\n *   position, except for the first one of the stream, which reports the\n *   slot's `confirmed_flush_lsn` and is thus ignored if it is at or past the\n *   start.\n *\n * Note that on an idle upstream, a walsender whose start is already\n * confirmed may not send a keepalive until other WAL is written or the\n * periodic keepalive (`wal_sender_timeout / 2`) is due, so the measured\n * time is an upper bound when completed by a keepalive.\n */\nexport class InitialDecodeTracker {\n  static async start(\n    lc: LogContext,\n    session: postgres.Sql,\n    slot: string,\n    lsn: bigint,\n  ): Promise<InitialDecodeTracker> {\n    let restartLSN: bigint | undefined;\n    let confirmedFlushLSN: bigint | undefined;\n    try {\n      // Replication sessions only support the simple query protocol, so the\n      // slot name is inlined as a literal.\n      const [row] = await session\n        .unsafe<{restartLSN: LSN | null; confirmedFlushLSN: LSN | null}[]>(\n          `SELECT restart_lsn AS \"restartLSN\",\n                  confirmed_flush_lsn AS \"confirmedFlushLSN\"\n             FROM pg_replication_slots\n             WHERE slot_name = ${lit(slot)}`,\n        )\n        .simple();\n      restartLSN = row?.restartLSN ? toBigInt(row.restartLSN) : undefined;\n      confirmedFlushLSN = row?.confirmedFlushLSN\n        ? toBigInt(row.confirmedFlushLSN)\n        : undefined;\n    } catch (e) {\n      lc.warn?.(`unable to read position of replication slot ${slot}`, e);\n    }\n    return new InitialDecodeTracker(\n      lc,\n      lsn,\n      restartLSN,\n      confirmedFlushLSN,\n      Date.now(),\n    );\n  }\n\n  readonly #lc: LogContext;\n  readonly #start: bigint;\n  readonly #restartLSN: bigint | undefined;\n  readonly #confirmedFlushLSN: bigint | undefined;\n  readonly #startTime: number;\n  #first = true;\n  #done = false;\n\n  constructor(\n    lc: LogContext,\n    lsn: bigint,\n    restartLSN: bigint | undefined,\n    confirmedFlushLSN: bigint | undefined,\n    startTime: number,\n  ) {\n    // The walsender silently starts at confirmed_flush_lsn if the\n    // requested lsn is behind it.\n    this.#start =\n      confirmedFlushLSN !== undefined && confirmedFlushLSN > lsn\n        ? confirmedFlushLSN\n        : lsn;\n    this.#lc = lc.withContext('decodeStart', fromBigInt(this.#start));\n    this.#restartLSN = restartLSN;\n    this.#confirmedFlushLSN = confirmedFlushLSN;\n    this.#startTime = startTime;\n\n    if (restartLSN !== undefined) {\n      this.#lc.info?.(\n        `decoding ${this.#start - restartLSN} bytes of WAL from ` +\n          `restart_lsn ${fromBigInt(restartLSN)} to the start of the stream`,\n      );\n    }\n  }\n\n  /**\n   * Observes a received message, returning the elapsed decode time (ms) if\n   * this message completed the initial decode.\n   */\n  observe([lsn, msg]: StreamMessage, now: number): number | undefined {\n    if (this.#done) {\n      return undefined;\n    }\n    const first = this.#first;\n    this.#first = false;\n    if (msg.tag === 'keepalive') {\n      if (\n        lsn < this.#start ||\n        (first && lsn === this.#confirmedFlushLSN) // reports confirmed_flush\n      ) {\n        return undefined;\n      }\n    }\n    this.#done = true;\n    const elapsed = now - this.#startTime;\n    initialDecodeHistogram().recordMs(elapsed);\n    this.#lc.info?.(\n      `upstream decoded the WAL to the start of the stream in ` +\n        `${elapsed} ms (${msg.tag} at ${fromBigInt(lsn)}` +\n        (this.#restartLSN === undefined\n          ? ')'\n          : `, ${this.#start - this.#restartLSN} bytes from restart_lsn)`),\n      {elapsed},\n    );\n    return elapsed;\n  }\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * An `inboundTimeoutOverrideMs` (when positive) replaces the default\n * `2x wal_sender_timeout` inbound threshold. The two timeouts protect\n * different parties with asymmetric false-positive costs: `wal_sender_timeout`\n * is the server reaping dead clients (a false positive there costs a cheap\n * reconnect), while the inbound watchdog tears down the stream and replays the\n * in-flight transaction — expensive, and self-defeating when the wal sender is\n * legitimately silent for long stretches (e.g. decoding through WAL from\n * unpublished tables, or assembling a large transaction that pgoutput only\n * sends at commit). Operators who don't own the server setting (e.g. managed\n * or operator-provisioned Postgres with an aggressive cluster-wide\n * `wal_sender_timeout`) can widen the client-side fuse independently.\n *\n * The override has no effect when `wal_sender_timeout` is disabled (0): the\n * server then sends no periodic keepalives, so inbound silence is normal even\n * on an idle healthy connection and any silence-based watchdog would\n * false-positive.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(\n  walSenderTimeoutMs: number,\n  inboundTimeoutOverrideMs?: number | undefined,\n): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.). A 0 / negative / NaN override\n    // falls back to the default rather than degenerating the threshold.\n    inboundTimeoutMs:\n      inboundTimeoutOverrideMs !== undefined && inboundTimeoutOverrideMs > 0\n        ? inboundTimeoutOverrideMs\n        : walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(Math.min(10 * 2 ** i, 1000));\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * This essentially reserves the replication slot for this process by marking\n * it as \"active\" in pg_replication_slots, allowing replication slot cleanup\n * logic to distinguish a new-but-initializing replication slot from one that\n * has been abandoned / orphaned.\n *\n * When initialization (e.g. initial-sync) has finished, the server should take\n * over the replication slot using pg_terminate_backend, which will close the\n * session.\n *\n * Note that caller should not close the supplied `session`, as that will\n * also terminate the dummy replication session.\n */\nexport async function keepSlotActiveUntilTakenOver(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  dummyPublication: string,\n  lsn: bigint,\n) {\n  const [readable, writeable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    [dummyPublication],\n    lsn,\n    1,\n  );\n  lc.info?.(`keeping ${slot} active until taken over ...`);\n  // Discard the streamed messages. Otherwise the paused socket backs up the\n  // wal_sender (e.g. with keepalives and logical messages) for the duration\n  // of the initial sync / restore, and a wal_sender blocked on writing to the\n  // client does not exit (and release the slot) when it is terminated by the\n  // takeover; it instead blocks on sending the termination error to the\n  // client, until signaled again.\n  readable.resume();\n  // Send periodic status messages upstream to indicate that the session is\n  // alive, but keep the confirmed_flush_lsn at the initial lsn.\n  const keepalive = setInterval(() => writeable.write(makeAck(lsn)), 10_000);\n\n  readable.on('close', async () => {\n    lc.info?.(`released initial reservation of ${slot}`);\n    clearInterval(keepalive);\n    try {\n      await session.end();\n      lc.info?.(`ended session`);\n    } catch (e) {\n      lc.warn?.(`error ending session`, e);\n    }\n  });\n  readable.once('error', e => {\n    lc.info?.(`closing replication session to ${session.options.host}`, e);\n    readable.destroy();\n  });\n  return readable;\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;;AAoBA,IAAM,6CAA6C;AAcnD,SAAgB,4BACd,IACA,iBACA;CACA,OAAO,SACL,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;AACF;AAEA,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAClB,0BAIC;CACD,MAAM,UAAU,4BAA4B,IAAI,eAAe;CAQ/D,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,oBAAoB,wBAAwB;CACvE,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,uBAChD,iBAAiB,OACpC,qBAAqB,qBAAqB,IACvC,KACA,4BACN,gGAEN;CAEA,MAAM,SAAS,MAAM,qBAAqB,MAAM,IAAI,SAAS,MAAM,GAAG;CACtE,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,wBAC5C,iBAAiB,uDAE3C;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,MAAM,MAAM,mBAAmB,IAAI,QAAQ,MAAM;GACjD,IAAI,KACF,OAAO,QAAQ,KAAK,gBAAgB;GAEtC,OAAO;EACT;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;AAEA,IAAM,wCAAwC;CAC5C;CAAK;CAAK;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAK;CAAM;CAAM;CAAM;AAClE;AAEA,SAAS,yBAAyB;CAChC,OAAO,qBAAqB,eAAe,yBAAyB;EAClE,aACE;EAEF,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;;;;;;;;;;;;;;;;;;AAuBA,IAAa,uBAAb,MAAa,qBAAqB;CAChC,aAAa,MACX,IACA,SACA,MACA,KAC+B;EAC/B,IAAI;EACJ,IAAI;EACJ,IAAI;GAGF,MAAM,CAAC,OAAO,MAAM,QACjB,OACC;;;iCAGuB,IAAI,IAAI,GACjC,EACC,OAAO;GACV,aAAa,KAAK,aAAa,SAAS,IAAI,UAAU,IAAI,KAAA;GAC1D,oBAAoB,KAAK,oBACrB,SAAS,IAAI,iBAAiB,IAC9B,KAAA;EACN,SAAS,GAAG;GACV,GAAG,OAAO,+CAA+C,QAAQ,CAAC;EACpE;EACA,OAAO,IAAI,qBACT,IACA,KACA,YACA,mBACA,KAAK,IAAI,CACX;CACF;CAEA;CACA;CACA;CACA;CACA;CACA,SAAS;CACT,QAAQ;CAER,YACE,IACA,KACA,YACA,mBACA,WACA;EAGA,KAAKC,SACH,sBAAsB,KAAA,KAAa,oBAAoB,MACnD,oBACA;EACN,KAAKD,MAAM,GAAG,YAAY,eAAe,WAAW,KAAKC,MAAM,CAAC;EAChE,KAAKC,cAAc;EACnB,KAAKC,qBAAqB;EAC1B,KAAKC,aAAa;EAElB,IAAI,eAAe,KAAA,GACjB,KAAKJ,IAAI,OACP,YAAY,KAAKC,SAAS,WAAW,iCACpB,WAAW,UAAU,EAAE,4BAC1C;CAEJ;;;;;CAMA,QAAQ,CAAC,KAAK,MAAqB,KAAiC;EAClE,IAAI,KAAKI,OACP;EAEF,MAAM,QAAQ,KAAKC;EACnB,KAAKA,SAAS;EACd,IAAI,IAAI,QAAQ;OAEZ,MAAM,KAAKL,UACV,SAAS,QAAQ,KAAKE,oBAEvB;EAAA;EAGJ,KAAKE,QAAQ;EACb,MAAM,UAAU,MAAM,KAAKD;EAC3B,uBAAuB,EAAE,SAAS,OAAO;EACzC,KAAKJ,IAAI,OACP,0DACK,QAAQ,OAAO,IAAI,IAAI,MAAM,WAAW,GAAG,OAC7C,KAAKE,gBAAgB,KAAA,IAClB,MACA,KAAK,KAAKD,SAAS,KAAKC,YAAY,4BAC1C,EAAC,QAAO,CACV;EACA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,SAAgB,uBACd,oBACA,0BAMA;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAMA,kBACE,6BAA6B,KAAA,KAAa,2BAA2B,IACjE,2BACA,qBAAqB;EAE3B,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,KAAK,IAAI,KAAK,KAAK,GAAG,GAAI,CAAC;IACvC;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,SACA,MACA,kBACA,KACA;CACA,MAAM,CAAC,UAAU,aAAa,MAAM,uBAClC,IACA,SACA,MACA,CAAC,gBAAgB,GACjB,KACA,CACF;CACA,GAAG,OAAO,WAAW,KAAK,6BAA6B;CAOvD,SAAS,OAAO;CAGhB,MAAM,YAAY,kBAAkB,UAAU,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAM;CAEzE,SAAS,GAAG,SAAS,YAAY;EAC/B,GAAG,OAAO,mCAAmC,MAAM;EACnD,cAAc,SAAS;EACvB,IAAI;GACF,MAAM,QAAQ,IAAI;GAClB,GAAG,OAAO,eAAe;EAC3B,SAAS,GAAG;GACV,GAAG,OAAO,wBAAwB,CAAC;EACrC;CACF,CAAC;CACD,SAAS,KAAK,UAAS,MAAK;EAC1B,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,QAAQ,CAAC;EACrE,SAAS,QAAQ;CACnB,CAAC;CACD,OAAO;AACT;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}

package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts

@@ -0,0 +1,16 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { PostgresDB } from '../../../types/pg.ts';
+import type { ShardID } from '../../../types/shards.ts';
+type Options = {
+    inactiveSlotCleanupTimeoutMs?: number | undefined;
+    pollIntervalMs?: number | undefined;
+};
+export declare class ReplicationSlotCleanupMonitor {
+    #private;
+    constructor(lc: LogContext, shard: ShardID, upstream: PostgresDB, upstreamPgServerVersionNum: number, currentSlot: string, { inactiveSlotCleanupTimeoutMs, pollIntervalMs }?: Options);
+    start(): void;
+    stop(): void;
+    getSlotsToCleanup(now: Date): Promise<string[]>;
+}
+export {};
+//# sourceMappingURL=replication-slot-cleanup-monitor.d.ts.map

package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replication-slot-cleanup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAItD,KAAK,OAAO,GAAG;IACb,4BAA4B,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClD,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAKF,qBAAa,6BAA6B;;IAkCxC,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,QAAQ,EAAE,UAAU,EACpB,0BAA0B,EAAE,MAAM,EAClC,WAAW,EAAE,MAAM,EACnB,EAAC,4BAA4B,EAAE,cAAc,EAAC,GAAE,OAAY,EAW7D;IAED,KAAK,SAYJ;IAED,IAAI,SAIH;IAiBK,iBAAiB,CAAC,GAAG,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAqEpD;CACF"}

package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js

@@ -0,0 +1,113 @@
+import { replicationSlotPrefix } from "./schema/shard.js";
+import "../../../types/pg-versions.js";
+import { dropInactiveSlotsAndReplicas } from "./replication-slots.js";
+//#region ../zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.ts
+var DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS = 300 * 1e3;
+var DEFAULT_POLL_INTERVAL_MS = 60 * 1e3;
+var ReplicationSlotCleanupMonitor = class {
+	#lc;
+	/** The shard constrains which slots are monitored (e.g. zero_0_*). */
+	#shard;
+	#upstream;
+	/**
+	* This slot that this task is using. Cleanup of slots is predicated on
+	* this slot remaining active, in order to guarantee at least one healthy
+	* slot in the shard.
+	*/
+	#currentSlot;
+	/**
+	* How long a slot is inactive (while the currentSlot is active) before it
+	* is eligible to be cleaned up. Because new slots are immediately connected
+	* to, even while an initial-sync or replica-restore is in progress, slots
+	* should only be inactive if no server is using it, or if a server was
+	* disconnected (e.g. a pg restart) and is in the process of reconnecting
+	* to it (see `keepSlotActiveUntilTakenOver`).
+	*
+	* As such, this timeout should be long enough to account for a server
+	* reconnecting to upstream. Note again that the cleanup monitor only runs
+	* while at least one replication slot is active, guaranteeing that upstream
+	* replication is working.
+	*/
+	#inactiveSlotCleanupTimeoutMs;
+	#pollIntervalMs;
+	/** Manually tracked inactive times for PG < 17. null for PG 17+ */
+	#firstInactiveTimes;
+	#pollTimer;
+	constructor(lc, shard, upstream, upstreamPgServerVersionNum, currentSlot, { inactiveSlotCleanupTimeoutMs, pollIntervalMs } = {}) {
+		this.#lc = lc.withContext("component", "replication-slot-cleanup-monitor");
+		this.#shard = shard;
+		this.#upstream = upstream;
+		this.#currentSlot = currentSlot;
+		this.#inactiveSlotCleanupTimeoutMs = inactiveSlotCleanupTimeoutMs ?? DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS;
+		this.#pollIntervalMs = pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+		this.#firstInactiveTimes = upstreamPgServerVersionNum >= 17e4 ? null : /* @__PURE__ */ new Map();
+	}
+	start() {
+		if (this.#pollTimer) {
+			this.#lc.warn?.(`monitor already started`);
+			return;
+		}
+		this.#pollTimer = setInterval(() => this.#poll(/* @__PURE__ */ new Date()).catch((e) => this.#lc.warn?.(`error checking replication slots`, e)), this.#pollIntervalMs);
+	}
+	stop() {
+		clearInterval(this.#pollTimer);
+		this.#pollTimer = void 0;
+		this.#firstInactiveTimes?.clear();
+	}
+	async #poll(now) {
+		if (!this.#pollTimer) return;
+		const slots = await this.getSlotsToCleanup(now);
+		if (slots.length) await dropInactiveSlotsAndReplicas(this.#lc, this.#upstream, this.#shard, slots);
+	}
+	async getSlotsToCleanup(now) {
+		const slotPoolPrefix = replicationSlotPrefix(this.#shard);
+		const slotsToCleanUp = [];
+		const trackedInactiveTimes = this.#firstInactiveTimes;
+		if (trackedInactiveTimes) {
+			const shardSlots = await this.#upstream`
+        SELECT slot_name AS slot, active FROM pg_replication_slots
+          WHERE slot_name LIKE ${slotPoolPrefix + "%"};
+      `;
+			for (const { slot, active } of shardSlots) {
+				if (slot === this.#currentSlot && !active) {
+					this.#lc.warn?.(`current slot ${this.#currentSlot} is not active. disabling cleanup.`);
+					this.stop();
+					return [];
+				}
+				if (active) trackedInactiveTimes.delete(slot);
+				else {
+					const inactiveSince = trackedInactiveTimes.get(slot);
+					if (!inactiveSince) trackedInactiveTimes.set(slot, now);
+					else if (now.getTime() - inactiveSince.getTime() >= this.#inactiveSlotCleanupTimeoutMs) {
+						this.#lc.info?.(`slot ${slot} has been inactive since ${inactiveSince.toISOString()} and is eligible for cleanup`);
+						slotsToCleanUp.push(slot);
+					}
+				}
+			}
+		} else {
+			const expiration = now.getTime() - this.#inactiveSlotCleanupTimeoutMs;
+			const inactiveSlots = await this.#upstream`
+        SELECT slot_name AS slot, inactive_since AS "inactiveSince" FROM pg_replication_slots
+          WHERE slot_name LIKE ${slotPoolPrefix + "%"}
+            AND active = false
+            AND inactive_since IS NOT NULL;
+      `;
+			for (const { slot, inactiveSince } of inactiveSlots) {
+				if (slot === this.#currentSlot) {
+					this.#lc.warn?.(`current slot ${this.#currentSlot} is not active. disabling cleanup.`);
+					this.stop();
+					return [];
+				}
+				if (inactiveSince <= expiration) {
+					this.#lc.info?.(`slot ${slot} has been inactive since ${new Date(inactiveSince).toISOString()} and is eligible for cleanup`);
+					slotsToCleanUp.push(slot);
+				}
+			}
+		}
+		return slotsToCleanUp;
+	}
+};
+//#endregion
+export { ReplicationSlotCleanupMonitor };
+
+//# sourceMappingURL=replication-slot-cleanup-monitor.js.map

package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"replication-slot-cleanup-monitor.js","names":["#lc","#shard","#upstream","#currentSlot","#inactiveSlotCleanupTimeoutMs","#pollIntervalMs","#firstInactiveTimes","#pollTimer","#poll"],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport type {PostgresDB} from '../../../types/pg.ts';\nimport type {ShardID} from '../../../types/shards.ts';\nimport {dropInactiveSlotsAndReplicas} from './replication-slots.ts';\nimport {replicationSlotPrefix} from './schema/shard.ts';\n\ntype Options = {\n  inactiveSlotCleanupTimeoutMs?: number | undefined;\n  pollIntervalMs?: number | undefined;\n};\n\nconst DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS = 5 * 60 * 1000;\nconst DEFAULT_POLL_INTERVAL_MS = 60 * 1000;\n\nexport class ReplicationSlotCleanupMonitor {\n  readonly #lc: LogContext;\n\n  /** The shard constrains which slots are monitored (e.g. zero_0_*). */\n  readonly #shard: ShardID;\n  readonly #upstream: PostgresDB;\n  /**\n   * This slot that this task is using. Cleanup of slots is predicated on\n   * this slot remaining active, in order to guarantee at least one healthy\n   * slot in the shard.\n   */\n  readonly #currentSlot: string;\n\n  /**\n   * How long a slot is inactive (while the currentSlot is active) before it\n   * is eligible to be cleaned up. Because new slots are immediately connected\n   * to, even while an initial-sync or replica-restore is in progress, slots\n   * should only be inactive if no server is using it, or if a server was\n   * disconnected (e.g. a pg restart) and is in the process of reconnecting\n   * to it (see `keepSlotActiveUntilTakenOver`).\n   *\n   * As such, this timeout should be long enough to account for a server\n   * reconnecting to upstream. Note again that the cleanup monitor only runs\n   * while at least one replication slot is active, guaranteeing that upstream\n   * replication is working.\n   */\n  readonly #inactiveSlotCleanupTimeoutMs: number;\n  readonly #pollIntervalMs: number;\n\n  /** Manually tracked inactive times for PG < 17. null for PG 17+ */\n  readonly #firstInactiveTimes: Map<string, Date> | null;\n\n  #pollTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    upstream: PostgresDB,\n    upstreamPgServerVersionNum: number,\n    currentSlot: string,\n    {inactiveSlotCleanupTimeoutMs, pollIntervalMs}: Options = {},\n  ) {\n    this.#lc = lc.withContext('component', 'replication-slot-cleanup-monitor');\n    this.#shard = shard;\n    this.#upstream = upstream;\n    this.#currentSlot = currentSlot;\n    this.#inactiveSlotCleanupTimeoutMs =\n      inactiveSlotCleanupTimeoutMs ?? DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS;\n    this.#pollIntervalMs = pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;\n    this.#firstInactiveTimes =\n      upstreamPgServerVersionNum >= PG_17 ? null : new Map();\n  }\n\n  start() {\n    if (this.#pollTimer) {\n      this.#lc.warn?.(`monitor already started`);\n      return;\n    }\n    this.#pollTimer = setInterval(\n      () =>\n        this.#poll(new Date()).catch(e =>\n          this.#lc.warn?.(`error checking replication slots`, e),\n        ),\n      this.#pollIntervalMs,\n    );\n  }\n\n  stop() {\n    clearInterval(this.#pollTimer);\n    this.#pollTimer = undefined;\n    this.#firstInactiveTimes?.clear();\n  }\n\n  async #poll(now: Date) {\n    if (!this.#pollTimer) {\n      return; // concurrently stopped\n    }\n    const slots = await this.getSlotsToCleanup(now);\n    if (slots.length) {\n      await dropInactiveSlotsAndReplicas(\n        this.#lc,\n        this.#upstream,\n        this.#shard,\n        slots,\n      );\n    }\n  }\n\n  async getSlotsToCleanup(now: Date): Promise<string[]> {\n    const slotPoolPrefix = replicationSlotPrefix(this.#shard);\n\n    const slotsToCleanUp: string[] = [];\n    const trackedInactiveTimes = this.#firstInactiveTimes;\n\n    if (trackedInactiveTimes) {\n      // For PG <17, each poll tracks and updates earliest time the slot was\n      // observed to be inactive.\n      const shardSlots = await this.#upstream<{slot: string; active: boolean}[]>\n      /*sql*/ `\n        SELECT slot_name AS slot, active FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n      `;\n      for (const {slot, active} of shardSlots) {\n        if (slot === this.#currentSlot && !active) {\n          this.#lc.warn?.(\n            `current slot ${this.#currentSlot} is not active. disabling cleanup.`,\n          );\n          this.stop();\n          return [];\n        }\n        if (active) {\n          trackedInactiveTimes.delete(slot);\n        } else {\n          const inactiveSince = trackedInactiveTimes.get(slot);\n          if (!inactiveSince) {\n            trackedInactiveTimes.set(slot, now);\n          } else if (\n            now.getTime() - inactiveSince.getTime() >=\n            this.#inactiveSlotCleanupTimeoutMs\n          ) {\n            this.#lc.info?.(\n              `slot ${slot} has been inactive since ${inactiveSince.toISOString()} and is eligible for cleanup`,\n            );\n            slotsToCleanUp.push(slot);\n          }\n        }\n      }\n    } else {\n      // PG 17+ is simpler as inactivity time is tracked in the table.\n      const expiration = now.getTime() - this.#inactiveSlotCleanupTimeoutMs;\n      const inactiveSlots = await this.#upstream<\n        {slot: string; inactiveSince: number}[]\n      > /*sql*/ `\n        SELECT slot_name AS slot, inactive_since AS \"inactiveSince\" FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'}\n            AND active = false\n            AND inactive_since IS NOT NULL;\n      `;\n      for (const {slot, inactiveSince} of inactiveSlots) {\n        if (slot === this.#currentSlot) {\n          this.#lc.warn?.(\n            `current slot ${this.#currentSlot} is not active. disabling cleanup.`,\n          );\n          this.stop();\n          return [];\n        }\n        // The slot has been inactive since at or before the expiration cutoff\n        // (now - timeout), i.e. inactive for at least the timeout duration.\n        if (inactiveSince <= expiration) {\n          this.#lc.info?.(\n            `slot ${slot} has been inactive since ${new Date(inactiveSince).toISOString()} and is eligible for cleanup`,\n          );\n          slotsToCleanUp.push(slot);\n        }\n      }\n    }\n    return slotsToCleanUp;\n  }\n}\n"],"mappings":";;;;AAYA,IAAM,2CAA2C,MAAS;AAC1D,IAAM,2BAA2B,KAAK;AAEtC,IAAa,gCAAb,MAA2C;CACzC;;CAGA;CACA;;;;;;CAMA;;;;;;;;;;;;;;CAeA;CACA;;CAGA;CAEA;CAEA,YACE,IACA,OACA,UACA,4BACA,aACA,EAAC,8BAA8B,mBAA2B,CAAC,GAC3D;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kCAAkC;EACzE,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,eAAe;EACpB,KAAKC,gCACH,gCAAgC;EAClC,KAAKC,kBAAkB,kBAAkB;EACzC,KAAKC,sBACH,8BAAA,OAAsC,uBAAO,IAAI,IAAI;CACzD;CAEA,QAAQ;EACN,IAAI,KAAKC,YAAY;GACnB,KAAKP,IAAI,OAAO,yBAAyB;GACzC;EACF;EACA,KAAKO,aAAa,kBAEd,KAAKC,sBAAM,IAAI,KAAK,CAAC,EAAE,OAAM,MAC3B,KAAKR,IAAI,OAAO,oCAAoC,CAAC,CACvD,GACF,KAAKK,eACP;CACF;CAEA,OAAO;EACL,cAAc,KAAKE,UAAU;EAC7B,KAAKA,aAAa,KAAA;EAClB,KAAKD,qBAAqB,MAAM;CAClC;CAEA,MAAME,MAAM,KAAW;EACrB,IAAI,CAAC,KAAKD,YACR;EAEF,MAAM,QAAQ,MAAM,KAAK,kBAAkB,GAAG;EAC9C,IAAI,MAAM,QACR,MAAM,6BACJ,KAAKP,KACL,KAAKE,WACL,KAAKD,QACL,KACF;CAEJ;CAEA,MAAM,kBAAkB,KAA8B;EACpD,MAAM,iBAAiB,sBAAsB,KAAKA,MAAM;EAExD,MAAM,iBAA2B,CAAC;EAClC,MAAM,uBAAuB,KAAKK;EAElC,IAAI,sBAAsB;GAGxB,MAAM,aAAa,MAAM,KAAKJ,SACtB;;iCAEmB,iBAAiB,IAAI;;GAEhD,KAAK,MAAM,EAAC,MAAM,YAAW,YAAY;IACvC,IAAI,SAAS,KAAKC,gBAAgB,CAAC,QAAQ;KACzC,KAAKH,IAAI,OACP,gBAAgB,KAAKG,aAAa,mCACpC;KACA,KAAK,KAAK;KACV,OAAO,CAAC;IACV;IACA,IAAI,QACF,qBAAqB,OAAO,IAAI;SAC3B;KACL,MAAM,gBAAgB,qBAAqB,IAAI,IAAI;KACnD,IAAI,CAAC,eACH,qBAAqB,IAAI,MAAM,GAAG;UAC7B,IACL,IAAI,QAAQ,IAAI,cAAc,QAAQ,KACtC,KAAKC,+BACL;MACA,KAAKJ,IAAI,OACP,QAAQ,KAAK,2BAA2B,cAAc,YAAY,EAAE,6BACtE;MACA,eAAe,KAAK,IAAI;KAC1B;IACF;GACF;EACF,OAAO;GAEL,MAAM,aAAa,IAAI,QAAQ,IAAI,KAAKI;GACxC,MAAM,gBAAgB,MAAM,KAAKF,SAEvB;;iCAEiB,iBAAiB,IAAI;;;;GAIhD,KAAK,MAAM,EAAC,MAAM,mBAAkB,eAAe;IACjD,IAAI,SAAS,KAAKC,cAAc;KAC9B,KAAKH,IAAI,OACP,gBAAgB,KAAKG,aAAa,mCACpC;KACA,KAAK,KAAK;KACV,OAAO,CAAC;IACV;IAGA,IAAI,iBAAiB,YAAY;KAC/B,KAAKH,IAAI,OACP,QAAQ,KAAK,2BAA2B,IAAI,KAAK,aAAa,EAAE,YAAY,EAAE,6BAChF;KACA,eAAe,KAAK,IAAI;IAC1B;GACF;EACF;EACA,OAAO;CACT;AACF"}

package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts

@@ -3,7 +3,8 @@ import type { LogContext } from '@rocicorp/logger';
 import type postgres from 'postgres';
 import { type PostgresDB } from '../../../types/pg.ts';
 import { type ShardID } from '../../../types/shards.ts';
-import { type BackupOptions } from './schema/shard.ts';
+import { type ReplicaStage } from './schema/replica-stage-enum.ts';
+import { type BackupOptions, type ReplicaState } from './schema/shard.ts';
 export type ReplicationSlot = {
     slot_name: string;
     consistent_point: string;
@@ -14,14 +15,35 @@ export type ReplicationSlotResult<T> = {
     slot: ReplicationSlot;
     capturedSnapshot: T;
     initialSession: Readable;
+    replica: ReplicaState;
+};
+export type ReservedSlot = {
+    slot: string;
+    reservation: Readable;
+    replica: ReplicaState;
 };
 export type CreateSlotSpec = {
     slotName: string;
     failover?: boolean;
     temporary?: boolean;
     lockTimeout?: number;
+    diagnosticsDB?: PostgresDB | undefined;
 };
-export declare function createReplicationSlot(lc: LogContext, session: postgres.Sql, { slotName, failover, temporary, lockTimeout, }: CreateSlotSpec): Promise<ReplicationSlot>;
+export declare const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60000;
+export declare function createReplicationSlot(lc: LogContext, session: postgres.Sql, { slotName, failover, temporary, lockTimeout, diagnosticsDB, }: CreateSlotSpec): Promise<ReplicationSlot>;
+/**
+ * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting
+ * on: the walsender's wait event and blocking pids, and the oldest
+ * transactions holding an xid. Slot creation waits for every such transaction
+ * on the instance (in all databases), each with its own lock_timeout, so a
+ * chain of overlapping transactions can exceed the client-side timeout
+ * without the lock_timeout ever firing.
+ *
+ * Query text is deliberately omitted since it can contain row values.
+ *
+ * Exported for testing.
+ */
+export declare function logSlotCreationBlockers(lc: LogContext, sql: PostgresDB, walsenderPID: number, slotName: string): Promise<void>;
 /**
  * Replica and slot creation involves two sessions for proper coordination
  * with other replica management logic:
@@ -61,22 +83,25 @@ export declare function createReplicationSlot(lc: LogContext, session: postgres.
  * PostgresChangeSource takes over the slot to process the stream in earnest.
  * It is also returned for cleanup in the case of failures (and for control
  * in unit tests).
+ *
+ * For replica's created in the `Restore` stage, the `generation` is
+ * initialized to an empty string instead of the slot's consistent point,
+ * since the slot is not being used for initial sync. It is the
+ * responsibiliy of the caller to call {@link initRestoreReplica} when
+ * the source replica has been selected for restoring, which will carry
+ * over the `generation` and other relevant values from the source.
  */
-export declare function createReplicaAndSlot<T>(lc: LogContext, sql: PostgresDB, sessionName: string, shard: ShardID, replicaID: string, failover: boolean, backupOptions: BackupOptions, captureSnapshot: (snapshot: string) => Promise<T>): Promise<ReplicationSlotResult<T>>;
+export declare function createReplicaAndSlot<T>(lc: LogContext, sql: PostgresDB, sessionName: string, shard: ShardID, epoch: number, replicaID: string, failover: boolean, backupOptions: BackupOptions, captureSnapshot: (snapshot: string) => Promise<T>, stage: ReplicaStage): Promise<ReplicationSlotResult<T>>;
 /**
- * Deletes "old" replicas (i.e. those with a lower rank than the current)
- * and attempts to drop replication slots that are not associated with any
- * replica.
+ * Claims the inactive slot of the given replica for use by this task, opening
+ * a replication session to mark it as active. The process should follow up by
+ * taking over the slot via the PostgresChangeSource.
  *
- * If a slot could not be dropped because there is still an active subscriber,
- * it will be reflected in the `draining` count that is returned. When there
- * are draining slots, the method should be retried until all orphaned slots
- * have been dropped.
+ * The replica is identified by its ID (and not just its slot) because rows of
+ * replicas whose slots were dropped outside of zero-cache's control may share
+ * the slot name.
  */
-export declare function dropOldReplicasAndSlots(lc: LogContext, sql: PostgresDB, shard: ShardID, beforeRank: bigint): Promise<{
-    dropped: number;
-    active: number;
-    draining: number;
-}>;
+export declare function claimSlotForResumption(lc: LogContext, sql: PostgresDB, shard: ShardID, sessionName: string, { id: replicaID, slot }: Pick<ReplicaState, 'id' | 'slot'>): Promise<ReservedSlot | null>;
+export declare function dropInactiveSlotsAndReplicas(lc: LogContext, sql: PostgresDB, shard: ShardID, slots: string[]): Promise<void>;
 export declare function slotPoolSuffix(n: number): string;
 //# sourceMappingURL=replication-slots.d.ts.map

package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAK1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAOtE,OAAO,EAKL,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,qBAAqB,CAAC,CAAC,IAAI;IACrC,IAAI,EAAE,eAAe,CAAC;IACtB,gBAAgB,EAAE,CAAC,CAAC;IACpB,cAAc,EAAE,QAAQ,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IAGjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAInB,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAoBF,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,EACE,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,WAAoC,GACrC,EAAE,cAAc,GAChB,OAAO,CAAC,eAAe,CAAC,CAwC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GAChD,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAuFnC;AAED;;;;;;;;;GASG;AACH,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC,CAY9D;AA+DD,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,UAUvC"}
+{"version":3,"file":"replication-slots.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAK1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAKrC,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAQtE,OAAO,EAGL,KAAK,YAAY,EAClB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAOL,KAAK,aAAa,EAClB,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,qBAAqB,CAAC,CAAC,IAAI;IACrC,IAAI,EAAE,eAAe,CAAC;IACtB,gBAAgB,EAAE,CAAC,CAAC;IACpB,cAAc,EAAE,QAAQ,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,QAAQ,CAAC;IACtB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IAGjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAInB,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,CAAC;IAIrB,aAAa,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;CACxC,CAAC;AAUF,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAUzD,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,EACE,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,WAAoC,EACpC,aAAa,GACd,EAAE,cAAc,GAChB,OAAO,CAAC,eAAe,CAAC,CAiD1B;AAKD;;;;;;;;;;;GAWG;AACH,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,iBAmCjB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,KAAK,EAAE,YAAY,GAClB,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAoHnC;AAED;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAC,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,GAAG,MAAM,CAAC,GACvD,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAsD9B;AAED,wBAAgB,4BAA4B,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EAAE,iBA+BhB;AAmFD,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,UAUvC"}