@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.40

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,QAA2C,MAAM,UAAU,CAAC;AAGnE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,yBAAyB,CAAC;AAExD,OAAO,EAAO,KAAK,IAAI,EAAE,KAAK,MAAM,EAAC,MAAM,8BAA8B,CAAC;AAK1E,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,qBAAqB,CAAC;AAIjD,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,EAAE,MAAM;IACX,OAAO,GAAG;QAAC,GAAG,EAAE,WAAW,CAAC;QAAC,aAAa,EAAE,OAAO,CAAA;KAAC;CACrD,CAAC;AAKF,KAAK,sBAAsB,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG;IAC3C,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,wBAAgB,2BAA2B,CACzC,EAAE,EAAE,UAAU,EACd,eAAe,EAAE,MAAM,oBAqBxB;AAED,wBAAsB,SAAS,CAC7B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,EAAE,EACtB,GAAG,EAAE,MAAM,EACX,8BAA8B,SAA6C,EAC3E,eAAe,SAAoB,EACnC,wBAAwB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC5C,OAAO,CAAC;IACT,QAAQ,EAAE,sBAAsB,CAAC,aAAa,CAAC,CAAC;IAChD,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;CACpB,CAAC,CAuID;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,wBAAgB,sBAAsB,CACpC,kBAAkB,EAAE,MAAM,EAC1B,wBAAwB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC5C;IACD,OAAO,EAAE,OAAO,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;CACzB,CA6BA;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,iBAAiB,EACjB,gBAAgB,EAChB,MAAM,GACP,EAAE;IACD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG;IAAC,WAAW,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAC,CAK5C;AA8DD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,IAAI,EAAE,MAAM,EACZ,gBAAgB,EAAE,MAAM,EACxB,GAAG,EAAE,MAAM,sCA8BZ"}
+{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAMrC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,QAA2C,MAAM,UAAU,CAAC;AAInE,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,mCAAmC,CAAC;AAExE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAc,KAAK,MAAM,EAAC,MAAM,gCAAgC,CAAC;AAExE,OAAO,EAAO,KAAK,IAAI,EAAE,KAAK,MAAM,EAAC,MAAM,8BAA8B,CAAC;AAK1E,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,qBAAqB,CAAC;AAEjD,eAAO,MAAM,0CAA0C,IAAI,CAAC;AAE5D,MAAM,MAAM,aAAa,GAAG;IAC1B,GAAG,EAAE,MAAM;IACX,OAAO,GAAG;QAAC,GAAG,EAAE,WAAW,CAAC;QAAC,aAAa,EAAE,OAAO,CAAA;KAAC;CACrD,CAAC;AAKF,KAAK,sBAAsB,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG;IAC3C,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,wBAAgB,2BAA2B,CACzC,EAAE,EAAE,UAAU,EACd,eAAe,EAAE,MAAM,oBAqBxB;AAED,wBAAsB,SAAS,CAC7B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,EAAE,EACtB,GAAG,EAAE,MAAM,EACX,8BAA8B,SAA6C,EAC3E,eAAe,SAAoB,EACnC,wBAAwB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC5C,OAAO,CAAC;IACT,QAAQ,EAAE,sBAAsB,CAAC,aAAa,CAAC,CAAC;IAChD,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;CACpB,CAAC,CA4ID;AAgBD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,oBAAoB;;IAC/B,OAAa,KAAK,CAChB,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,oBAAoB,CAAC,CA4B/B;IAUD,YACE,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,iBAAiB,EAAE,MAAM,GAAG,SAAS,EACrC,SAAS,EAAE,MAAM,EAmBlB;IAED;;;OAGG;IACH,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CA0BlE;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,wBAAgB,sBAAsB,CACpC,kBAAkB,EAAE,MAAM,EAC1B,wBAAwB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC5C;IACD,OAAO,EAAE,OAAO,CAAC;IACjB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;CACzB,CA6BA;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,uBAAuB,CAAC,EACtC,iBAAiB,EACjB,gBAAgB,EAChB,MAAM,GACP,EAAE;IACD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG;IAAC,WAAW,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAC,CAK5C;AAaD,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,EAAE,EACtB,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,MAAuD,kDAyCrE;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,qBAAsB,SAAQ,QAAQ;;IAMjD,YAAY,MAAM,EAAE,MAAM,EAqBzB;IAEQ,QAAQ,CACf,KAAK,EAAE,KAAK,GAAG,IAAI,EACnB,QAAQ,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI,KAAK,IAAI,GACvC,IAAI,CAgBN;IAEK,GAAG,CAAC,SAAS,SAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAc1C;CACF;AAED,wBAAsB,UAAU,CAAC,OAAO,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CASlE;AAED,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC;IACrC,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC;IACrC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,GAAG,CAAC,EACA,OAAO,GACP,MAAM,GACN;QAAC,kBAAkB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;KAAC,GAC1C,SAAS,CAAC;CACf,CAAC;AAEF,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,QAAQ,CAAC,GAAG,GACpB,kBAAkB,CAkBpB;AAED;;;;;;;;GAQG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,QAAQ,CAAC,CAqCnB;AAyBD,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAU3C"}

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js

@@ -1,7 +1,10 @@
 import { sleep } from "../../../../../../shared/src/sleep.js";
+import { childWorker } from "../../../../types/processes.js";
+import { getOrCreateHistogram } from "../../../../observability/metrics.js";
 import { id, lit } from "../../../../types/sql.js";
 import "../../../../types/pg.js";
-import { fromBigInt } from "../lsn.js";
+import { SLOT_KEEPER_URL } from "../../../../server/worker-urls.js";
+import { fromBigInt, toBigInt } from "../lsn.js";
 import { Subscription } from "../../../../types/subscription.js";
 import { pipe } from "../../../../types/streams.js";
 import { AutoResetSignal } from "../../../change-streamer/schema/tables.js";
@@ -10,8 +13,7 @@ import { PgoutputParser } from "./pgoutput-parser.js";
 import { defu } from "defu";
 import postgres from "postgres";
 import { PG_ADMIN_SHUTDOWN, PG_OBJECT_IN_USE, PG_OBJECT_NOT_IN_PREREQUISITE_STATE } from "@drdgvhbh/postgres-error-codes";
-//#region ../zero-cache/src/services/change-source/pg/logical-replication/stream.ts
-var DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;
+import { Readable } from "node:stream";
 function createReplicationSessionFor(db, applicationName) {
 	return postgres(defu({
 		max: 1,
@@ -24,13 +26,14 @@ function createReplicationSessionFor(db, applicationName) {
 		}
 	}, db.options));
 }
-async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE, applicationName = "zero-replicator", inboundTimeoutOverrideMs) {
+async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSlotActive = 5, applicationName = "zero-replicator", inboundTimeoutOverrideMs) {
 	const session = createReplicationSessionFor(db, applicationName);
 	const [{ walSenderTimeoutMs }] = await session`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 
         AS "walSenderTimeoutMs" FROM pg_settings
         WHERE name = 'wal_sender_timeout'`.simple();
 	const { enabled: livenessEnabled, manualKeepaliveTimeout, inboundTimeoutMs, timerIntervalMs } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);
 	lc.info?.(livenessEnabled ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. Inbound timeout: ${inboundTimeoutMs}ms` + (inboundTimeoutMs === walSenderTimeoutMs * 2 ? "" : " (configured override)") : "wal_sender_timeout is disabled (0); skipping manual keepalives and inbound liveness detection.");
+	const decode = await InitialDecodeTracker.start(lc, session, slot, lsn);
 	const [readable, writable] = await startReplicationStream(lc, session, slot, publications, lsn, retriesIfReplicationSlotActive + 1);
 	let lastAckTime = Date.now();
 	function sendAck(lsn) {
@@ -71,7 +74,9 @@ async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSl
 		sink: messages,
 		parse: (buffer) => {
 			lastReceivedTime = Date.now();
-			return parseStreamMessage(lc, buffer, parser);
+			const msg = parseStreamMessage(lc, buffer, parser);
+			if (msg) decode.observe(msg, lastReceivedTime);
+			return msg;
 		},
 		bufferMessages: 5
 	});
@@ -80,6 +85,100 @@ async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSl
 		acks: { push: sendAck }
 	};
 }
+var INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S = [
+	.1,
+	.5,
+	1,
+	2,
+	5,
+	10,
+	30,
+	60,
+	120,
+	300,
+	600,
+	1200,
+	2400,
+	3600,
+	7200
+];
+function initialDecodeHistogram() {
+	return getOrCreateHistogram("replication", "stream_initial_decode", {
+		description: "Time from START_REPLICATION until upstream has decoded the WAL between the slot's restart_lsn and the start of the stream",
+		unit: "s",
+		bucketBoundaries: INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S
+	});
+}
+/**
+* Measures how long upstream takes to decode the WAL between the slot's
+* `restart_lsn` and the start of the stream, during which the stream
+* delivers nothing but keepalives. This span can be large, e.g. for a slot
+* that was created (or last acked) long before the requested start LSN.
+*
+* The walsender streams from `max(lsn, confirmed_flush_lsn)`, but must
+* decode from `restart_lsn` to rebuild its transaction state. The decode is
+* considered complete upon the first message at or past the start:
+* * any XLogData message, since only transactions committed after the
+*   start are sent, or
+* * a keepalive at or past the start. Keepalives report the decoding
+*   position, except for the first one of the stream, which reports the
+*   slot's `confirmed_flush_lsn` and is thus ignored if it is at or past the
+*   start.
+*
+* Note that on an idle upstream, a walsender whose start is already
+* confirmed may not send a keepalive until other WAL is written or the
+* periodic keepalive (`wal_sender_timeout / 2`) is due, so the measured
+* time is an upper bound when completed by a keepalive.
+*/
+var InitialDecodeTracker = class InitialDecodeTracker {
+	static async start(lc, session, slot, lsn) {
+		let restartLSN;
+		let confirmedFlushLSN;
+		try {
+			const [row] = await session.unsafe(`SELECT restart_lsn AS "restartLSN",
+                  confirmed_flush_lsn AS "confirmedFlushLSN"
+             FROM pg_replication_slots
+             WHERE slot_name = ${lit(slot)}`).simple();
+			restartLSN = row?.restartLSN ? toBigInt(row.restartLSN) : void 0;
+			confirmedFlushLSN = row?.confirmedFlushLSN ? toBigInt(row.confirmedFlushLSN) : void 0;
+		} catch (e) {
+			lc.warn?.(`unable to read position of replication slot ${slot}`, e);
+		}
+		return new InitialDecodeTracker(lc, lsn, restartLSN, confirmedFlushLSN, Date.now());
+	}
+	#lc;
+	#start;
+	#restartLSN;
+	#confirmedFlushLSN;
+	#startTime;
+	#first = true;
+	#done = false;
+	constructor(lc, lsn, restartLSN, confirmedFlushLSN, startTime) {
+		this.#start = confirmedFlushLSN !== void 0 && confirmedFlushLSN > lsn ? confirmedFlushLSN : lsn;
+		this.#lc = lc.withContext("decodeStart", fromBigInt(this.#start));
+		this.#restartLSN = restartLSN;
+		this.#confirmedFlushLSN = confirmedFlushLSN;
+		this.#startTime = startTime;
+		if (restartLSN !== void 0) this.#lc.info?.(`decoding ${this.#start - restartLSN} bytes of WAL from restart_lsn ${fromBigInt(restartLSN)} to the start of the stream`);
+	}
+	/**
+	* Observes a received message, returning the elapsed decode time (ms) if
+	* this message completed the initial decode.
+	*/
+	observe([lsn, msg], now) {
+		if (this.#done) return;
+		const first = this.#first;
+		this.#first = false;
+		if (msg.tag === "keepalive") {
+			if (lsn < this.#start || first && lsn === this.#confirmedFlushLSN) return;
+		}
+		this.#done = true;
+		const elapsed = now - this.#startTime;
+		initialDecodeHistogram().recordMs(elapsed);
+		this.#lc.info?.(`upstream decoded the WAL to the start of the stream in ${elapsed} ms (${msg.tag} at ${fromBigInt(lsn)}` + (this.#restartLSN === void 0 ? ")" : `, ${this.#start - this.#restartLSN} bytes from restart_lsn)`), { elapsed });
+		return elapsed;
+	}
+};
 /**
 * Derives the timings for the replication liveness timer from the server's
 * `wal_sender_timeout` (in milliseconds).
@@ -163,7 +262,7 @@ function evaluateInboundLiveness({ sinceLastReceived, inboundTimeoutMs, queued }
 function formatPublicationNames(publications) {
 	return publications.map((p) => lit(p).slice(1, -1)).join(",");
 }
-async function startReplicationStream(lc, session, slot, publications, lsn, maxAttempts) {
+async function startReplicationStream(lc, session, slot, publications, lsn, maxAttempts = 6) {
 	for (let i = 0; i < maxAttempts; i++) try {
 		const stream = session.unsafe(`START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (
         proto_version '1',
@@ -175,7 +274,7 @@ async function startReplicationStream(lc, session, slot, publications, lsn, maxA
 		if (e instanceof postgres.PostgresError) {
 			if (e.code === PG_OBJECT_IN_USE) {
 				lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);
-				await sleep(10);
+				await sleep(Math.min(10 * 2 ** i, 1e3));
 				continue;
 			}
 			if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {
@@ -204,25 +303,109 @@ async function startReplicationStream(lc, session, slot, publications, lsn, maxA
 * Note that caller should not close the supplied `session`, as that will
 * also terminate the dummy replication session.
 */
-async function keepSlotActiveUntilTakenOver(lc, session, slot, dummyPublication, lsn) {
-	const [readable, writeable] = await startReplicationStream(lc, session, slot, [dummyPublication], lsn, 1);
-	lc.info?.(`keeping ${slot} active until taken over ...`);
-	const keepalive = setInterval(() => writeable.write(makeAck(lsn)), 1e4);
-	readable.on("close", async () => {
-		lc.info?.(`released initial reservation of ${slot}`);
-		clearInterval(keepalive);
-		try {
-			await session.end();
-			lc.info?.(`ended session`);
-		} catch (e) {
-			lc.warn?.(`error ending session`, e);
+var SlotReservationHandle = class extends Readable {
+	#worker;
+	#closedPromise;
+	#stopped = false;
+	#sigtermTimer;
+	constructor(worker) {
+		super({ read() {} });
+		this.#worker = worker;
+		this.#closedPromise = new Promise((resolve) => {
+			worker.once("close", () => {
+				clearTimeout(this.#sigtermTimer);
+				resolve();
+			});
+		});
+		this.on("error", () => {});
+		worker.on("close", () => {
+			if (!this.destroyed) this.destroy();
+		});
+		worker.on("error", (err) => {
+			if (!this.destroyed) this.destroy(err instanceof Error ? err : new Error(String(err)));
+		});
+	}
+	_destroy(error, callback) {
+		if (!this.#stopped) {
+			this.#stopped = true;
+			try {
+				this.#worker.send(["stop", {}]);
+			} catch {}
+			this.#sigtermTimer = setTimeout(() => {
+				try {
+					this.#worker.kill("SIGTERM");
+				} catch {}
+			}, 5e3);
+			this.#sigtermTimer.unref();
+		}
+		callback(error);
+	}
+	async end(timeoutMs = 5e3) {
+		this.destroy();
+		if (!await Promise.race([this.#closedPromise.then(() => true), sleep(timeoutMs).then(() => false)])) {
+			try {
+				this.#worker.kill("SIGKILL");
+			} catch {}
+			await Promise.race([this.#closedPromise, sleep(2e3)]);
 		}
+	}
+};
+async function endSession(session) {
+	if (!session) return;
+	if ("end" in session && typeof session.end === "function") await session.end();
+	else session.destroy();
+}
+function extractConnectionConfig(session) {
+	const opts = session.options;
+	return {
+		host: opts.host,
+		port: opts.port,
+		path: opts.path,
+		database: opts.database,
+		user: opts.user,
+		pass: opts.pass,
+		ssl: typeof opts.ssl === "object" && opts.ssl !== null ? { rejectUnauthorized: opts.ssl.rejectUnauthorized } : opts.ssl
+	};
+}
+/**
+* Starts a "dummy" replication session for the specified `slot` which:
+* * does not consume any records
+* * keeps the session alive by sending status messages at the starting LSN
+*
+* Runs in a separate process with its own event loop and connection reconnect
+* loop to prevent slot drop races during heavy initial sync table copy or
+* SQLite index building.
+*/
+async function keepSlotActiveUntilTakenOver(lc, config) {
+	const { slot, lsn } = config;
+	lc.info?.(`spawning slot-keeper process for ${slot} at lsn ${lsn} ...`);
+	const worker = childWorker(SLOT_KEEPER_URL, process.env);
+	worker.onMessageType("init", () => {
+		worker.send(["start", config]);
 	});
-	readable.once("error", (e) => {
-		lc.info?.(`closing replication session to ${session.options.host}`, e);
-		readable.destroy();
+	await new Promise((resolve, reject) => {
+		const onReady = () => {
+			cleanup();
+			resolve();
+		};
+		const onError = (err) => {
+			cleanup();
+			reject(err);
+		};
+		const onClose = (code) => {
+			cleanup();
+			reject(/* @__PURE__ */ new Error(`slot-keeper for ${slot} exited with code ${code} before becoming ready`));
+		};
+		function cleanup() {
+			worker.off("error", onError);
+			worker.off("close", onClose);
+		}
+		worker.onceMessageType("ready", onReady);
+		worker.once("error", onError);
+		worker.once("close", onClose);
 	});
-	return readable;
+	lc.info?.(`slot-keeper active for ${slot}`);
+	return new SlotReservationHandle(worker);
 }
 function parseStreamMessage(lc, buffer, parser) {
 	if (buffer[0] !== 119 && buffer[0] !== 107) {
@@ -247,6 +430,6 @@ function makeAck(lsn) {
 	return x;
 }
 //#endregion
-export { createReplicationSessionFor, keepSlotActiveUntilTakenOver, subscribe };
+export { createReplicationSessionFor, endSession, extractConnectionConfig, keepSlotActiveUntilTakenOver, makeAck, startReplicationStream, subscribe };
 
 //# sourceMappingURL=stream.js.map

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport function createReplicationSessionFor(\n  db: PostgresDB,\n  applicationName: string,\n) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n  inboundTimeoutOverrideMs?: number | undefined,\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = createReplicationSessionFor(db, applicationName);\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. ` +\n          `Inbound timeout: ${inboundTimeoutMs}ms` +\n          (inboundTimeoutMs === walSenderTimeoutMs * 2\n            ? ''\n            : ' (configured override)')\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> inbound timeout ${inboundTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      return parseStreamMessage(lc, buffer, parser);\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * An `inboundTimeoutOverrideMs` (when positive) replaces the default\n * `2x wal_sender_timeout` inbound threshold. The two timeouts protect\n * different parties with asymmetric false-positive costs: `wal_sender_timeout`\n * is the server reaping dead clients (a false positive there costs a cheap\n * reconnect), while the inbound watchdog tears down the stream and replays the\n * in-flight transaction — expensive, and self-defeating when the wal sender is\n * legitimately silent for long stretches (e.g. decoding through WAL from\n * unpublished tables, or assembling a large transaction that pgoutput only\n * sends at commit). Operators who don't own the server setting (e.g. managed\n * or operator-provisioned Postgres with an aggressive cluster-wide\n * `wal_sender_timeout`) can widen the client-side fuse independently.\n *\n * The override has no effect when `wal_sender_timeout` is disabled (0): the\n * server then sends no periodic keepalives, so inbound silence is normal even\n * on an idle healthy connection and any silence-based watchdog would\n * false-positive.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(\n  walSenderTimeoutMs: number,\n  inboundTimeoutOverrideMs?: number | undefined,\n): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.). A 0 / negative / NaN override\n    // falls back to the default rather than degenerating the threshold.\n    inboundTimeoutMs:\n      inboundTimeoutOverrideMs !== undefined && inboundTimeoutOverrideMs > 0\n        ? inboundTimeoutOverrideMs\n        : walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * This essentially reserves the replication slot for this process by marking\n * it as \"active\" in pg_replication_slots, allowing replication slot cleanup\n * logic to distinguish a new-but-initializing replication slot from one that\n * has been abandoned / orphaned.\n *\n * When initialization (e.g. initial-sync) has finished, the server should take\n * over the replication slot using pg_terminate_backend, which will close the\n * session.\n *\n * Note that caller should not close the supplied `session`, as that will\n * also terminate the dummy replication session.\n */\nexport async function keepSlotActiveUntilTakenOver(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  dummyPublication: string,\n  lsn: bigint,\n) {\n  const [readable, writeable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    [dummyPublication],\n    lsn,\n    1,\n  );\n  lc.info?.(`keeping ${slot} active until taken over ...`);\n  // Send periodic status messages upstream to indicate that the session is\n  // alive, but keep the confirmed_flush_lsn at the initial lsn.\n  const keepalive = setInterval(() => writeable.write(makeAck(lsn)), 10_000);\n\n  readable.on('close', async () => {\n    lc.info?.(`released initial reservation of ${slot}`);\n    clearInterval(keepalive);\n    try {\n      await session.end();\n      lc.info?.(`ended session`);\n    } catch (e) {\n      lc.warn?.(`error ending session`, e);\n    }\n  });\n  readable.once('error', e => {\n    lc.info?.(`closing replication session to ${session.options.host}`, e);\n    readable.destroy();\n  });\n  return readable;\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;AAmBA,IAAM,6CAA6C;AAcnD,SAAgB,4BACd,IACA,iBACA;CACA,OAAO,SACL,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;AACF;AAEA,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAClB,0BAIC;CACD,MAAM,UAAU,4BAA4B,IAAI,eAAe;CAQ/D,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,oBAAoB,wBAAwB;CACvE,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,uBAChD,iBAAiB,OACpC,qBAAqB,qBAAqB,IACvC,KACA,4BACN,gGAEN;CAEA,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,wBAC5C,iBAAiB,uDAE3C;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,OAAO,mBAAmB,IAAI,QAAQ,MAAM;EAC9C;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,SAAgB,uBACd,oBACA,0BAMA;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAMA,kBACE,6BAA6B,KAAA,KAAa,2BAA2B,IACjE,2BACA,qBAAqB;EAE3B,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,SACA,MACA,kBACA,KACA;CACA,MAAM,CAAC,UAAU,aAAa,MAAM,uBAClC,IACA,SACA,MACA,CAAC,gBAAgB,GACjB,KACA,CACF;CACA,GAAG,OAAO,WAAW,KAAK,6BAA6B;CAGvD,MAAM,YAAY,kBAAkB,UAAU,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAM;CAEzE,SAAS,GAAG,SAAS,YAAY;EAC/B,GAAG,OAAO,mCAAmC,MAAM;EACnD,cAAc,SAAS;EACvB,IAAI;GACF,MAAM,QAAQ,IAAI;GAClB,GAAG,OAAO,eAAe;EAC3B,SAAS,GAAG;GACV,GAAG,OAAO,wBAAwB,CAAC;EACrC;CACF,CAAC;CACD,SAAS,KAAK,UAAS,MAAK;EAC1B,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,QAAQ,CAAC;EACrE,SAAS,QAAQ;CACnB,CAAC;CACD,OAAO;AACT;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}
+{"version":3,"file":"stream.js","names":["#lc","#start","#restartLSN","#confirmedFlushLSN","#startTime","#done","#first","#worker","#closedPromise","#sigtermTimer","#stopped"],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {Readable} from 'node:stream';\nimport {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {getOrCreateHistogram} from '../../../../observability/metrics.ts';\nimport type {SlotKeeperConfig} from '../../../../server/slot-keeper.ts';\nimport {SLOT_KEEPER_URL} from '../../../../server/worker-urls.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {childWorker, type Worker} from '../../../../types/processes.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt, toBigInt, type LSN} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nexport const DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport function createReplicationSessionFor(\n  db: PostgresDB,\n  applicationName: string,\n) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n  inboundTimeoutOverrideMs?: number | undefined,\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = createReplicationSessionFor(db, applicationName);\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. ` +\n          `Inbound timeout: ${inboundTimeoutMs}ms` +\n          (inboundTimeoutMs === walSenderTimeoutMs * 2\n            ? ''\n            : ' (configured override)')\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const decode = await InitialDecodeTracker.start(lc, session, slot, lsn);\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> inbound timeout ${inboundTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      const msg = parseStreamMessage(lc, buffer, parser);\n      if (msg) {\n        decode.observe(msg, lastReceivedTime);\n      }\n      return msg;\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\nconst INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S = [\n  0.1, 0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200,\n];\n\nfunction initialDecodeHistogram() {\n  return getOrCreateHistogram('replication', 'stream_initial_decode', {\n    description:\n      'Time from START_REPLICATION until upstream has decoded the WAL ' +\n      \"between the slot's restart_lsn and the start of the stream\",\n    unit: 's',\n    bucketBoundaries: INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n/**\n * Measures how long upstream takes to decode the WAL between the slot's\n * `restart_lsn` and the start of the stream, during which the stream\n * delivers nothing but keepalives. This span can be large, e.g. for a slot\n * that was created (or last acked) long before the requested start LSN.\n *\n * The walsender streams from `max(lsn, confirmed_flush_lsn)`, but must\n * decode from `restart_lsn` to rebuild its transaction state. The decode is\n * considered complete upon the first message at or past the start:\n * * any XLogData message, since only transactions committed after the\n *   start are sent, or\n * * a keepalive at or past the start. Keepalives report the decoding\n *   position, except for the first one of the stream, which reports the\n *   slot's `confirmed_flush_lsn` and is thus ignored if it is at or past the\n *   start.\n *\n * Note that on an idle upstream, a walsender whose start is already\n * confirmed may not send a keepalive until other WAL is written or the\n * periodic keepalive (`wal_sender_timeout / 2`) is due, so the measured\n * time is an upper bound when completed by a keepalive.\n */\nexport class InitialDecodeTracker {\n  static async start(\n    lc: LogContext,\n    session: postgres.Sql,\n    slot: string,\n    lsn: bigint,\n  ): Promise<InitialDecodeTracker> {\n    let restartLSN: bigint | undefined;\n    let confirmedFlushLSN: bigint | undefined;\n    try {\n      // Replication sessions only support the simple query protocol, so the\n      // slot name is inlined as a literal.\n      const [row] = await session\n        .unsafe<{restartLSN: LSN | null; confirmedFlushLSN: LSN | null}[]>(\n          `SELECT restart_lsn AS \"restartLSN\",\n                  confirmed_flush_lsn AS \"confirmedFlushLSN\"\n             FROM pg_replication_slots\n             WHERE slot_name = ${lit(slot)}`,\n        )\n        .simple();\n      restartLSN = row?.restartLSN ? toBigInt(row.restartLSN) : undefined;\n      confirmedFlushLSN = row?.confirmedFlushLSN\n        ? toBigInt(row.confirmedFlushLSN)\n        : undefined;\n    } catch (e) {\n      lc.warn?.(`unable to read position of replication slot ${slot}`, e);\n    }\n    return new InitialDecodeTracker(\n      lc,\n      lsn,\n      restartLSN,\n      confirmedFlushLSN,\n      Date.now(),\n    );\n  }\n\n  readonly #lc: LogContext;\n  readonly #start: bigint;\n  readonly #restartLSN: bigint | undefined;\n  readonly #confirmedFlushLSN: bigint | undefined;\n  readonly #startTime: number;\n  #first = true;\n  #done = false;\n\n  constructor(\n    lc: LogContext,\n    lsn: bigint,\n    restartLSN: bigint | undefined,\n    confirmedFlushLSN: bigint | undefined,\n    startTime: number,\n  ) {\n    // The walsender silently starts at confirmed_flush_lsn if the\n    // requested lsn is behind it.\n    this.#start =\n      confirmedFlushLSN !== undefined && confirmedFlushLSN > lsn\n        ? confirmedFlushLSN\n        : lsn;\n    this.#lc = lc.withContext('decodeStart', fromBigInt(this.#start));\n    this.#restartLSN = restartLSN;\n    this.#confirmedFlushLSN = confirmedFlushLSN;\n    this.#startTime = startTime;\n\n    if (restartLSN !== undefined) {\n      this.#lc.info?.(\n        `decoding ${this.#start - restartLSN} bytes of WAL from ` +\n          `restart_lsn ${fromBigInt(restartLSN)} to the start of the stream`,\n      );\n    }\n  }\n\n  /**\n   * Observes a received message, returning the elapsed decode time (ms) if\n   * this message completed the initial decode.\n   */\n  observe([lsn, msg]: StreamMessage, now: number): number | undefined {\n    if (this.#done) {\n      return undefined;\n    }\n    const first = this.#first;\n    this.#first = false;\n    if (msg.tag === 'keepalive') {\n      if (\n        lsn < this.#start ||\n        (first && lsn === this.#confirmedFlushLSN) // reports confirmed_flush\n      ) {\n        return undefined;\n      }\n    }\n    this.#done = true;\n    const elapsed = now - this.#startTime;\n    initialDecodeHistogram().recordMs(elapsed);\n    this.#lc.info?.(\n      `upstream decoded the WAL to the start of the stream in ` +\n        `${elapsed} ms (${msg.tag} at ${fromBigInt(lsn)}` +\n        (this.#restartLSN === undefined\n          ? ')'\n          : `, ${this.#start - this.#restartLSN} bytes from restart_lsn)`),\n      {elapsed},\n    );\n    return elapsed;\n  }\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * An `inboundTimeoutOverrideMs` (when positive) replaces the default\n * `2x wal_sender_timeout` inbound threshold. The two timeouts protect\n * different parties with asymmetric false-positive costs: `wal_sender_timeout`\n * is the server reaping dead clients (a false positive there costs a cheap\n * reconnect), while the inbound watchdog tears down the stream and replays the\n * in-flight transaction — expensive, and self-defeating when the wal sender is\n * legitimately silent for long stretches (e.g. decoding through WAL from\n * unpublished tables, or assembling a large transaction that pgoutput only\n * sends at commit). Operators who don't own the server setting (e.g. managed\n * or operator-provisioned Postgres with an aggressive cluster-wide\n * `wal_sender_timeout`) can widen the client-side fuse independently.\n *\n * The override has no effect when `wal_sender_timeout` is disabled (0): the\n * server then sends no periodic keepalives, so inbound silence is normal even\n * on an idle healthy connection and any silence-based watchdog would\n * false-positive.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(\n  walSenderTimeoutMs: number,\n  inboundTimeoutOverrideMs?: number | undefined,\n): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.). A 0 / negative / NaN override\n    // falls back to the default rather than degenerating the threshold.\n    inboundTimeoutMs:\n      inboundTimeoutOverrideMs !== undefined && inboundTimeoutOverrideMs > 0\n        ? inboundTimeoutOverrideMs\n        : walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nexport async function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE + 1,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(Math.min(10 * 2 ** i, 1000));\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * This essentially reserves the replication slot for this process by marking\n * it as \"active\" in pg_replication_slots, allowing replication slot cleanup\n * logic to distinguish a new-but-initializing replication slot from one that\n * has been abandoned / orphaned.\n *\n * When initialization (e.g. initial-sync) has finished, the server should take\n * over the replication slot using pg_terminate_backend, which will close the\n * session.\n *\n * Note that caller should not close the supplied `session`, as that will\n * also terminate the dummy replication session.\n */\nexport class SlotReservationHandle extends Readable {\n  readonly #worker: Worker;\n  readonly #closedPromise: Promise<void>;\n  #stopped = false;\n  #sigtermTimer?: NodeJS.Timeout | undefined;\n\n  constructor(worker: Worker) {\n    super({read() {}});\n    this.#worker = worker;\n    this.#closedPromise = new Promise(resolve => {\n      worker.once('close', () => {\n        clearTimeout(this.#sigtermTimer);\n        resolve();\n      });\n    });\n    // Suppress unhandled error events if destroyed with an error\n    this.on('error', () => {});\n    worker.on('close', () => {\n      if (!this.destroyed) {\n        this.destroy();\n      }\n    });\n    worker.on('error', err => {\n      if (!this.destroyed) {\n        this.destroy(err instanceof Error ? err : new Error(String(err)));\n      }\n    });\n  }\n\n  override _destroy(\n    error: Error | null,\n    callback: (error?: Error | null) => void,\n  ): void {\n    if (!this.#stopped) {\n      this.#stopped = true;\n      try {\n        this.#worker.send(['stop', {}]);\n      } catch {\n        // ignore\n      }\n      this.#sigtermTimer = setTimeout(() => {\n        try {\n          this.#worker.kill('SIGTERM');\n        } catch {}\n      }, 5_000);\n      this.#sigtermTimer.unref();\n    }\n    callback(error);\n  }\n\n  async end(timeoutMs = 5_000): Promise<void> {\n    this.destroy();\n    const closed = await Promise.race([\n      this.#closedPromise.then(() => true),\n      sleep(timeoutMs).then(() => false),\n    ]);\n    if (!closed) {\n      try {\n        this.#worker.kill('SIGKILL');\n      } catch {}\n      // Await worker closure after forced kill so that the replication session\n      // is completely terminated before any subsequent slot drop attempt.\n      await Promise.race([this.#closedPromise, sleep(2_000)]);\n    }\n  }\n}\n\nexport async function endSession(session?: Readable): Promise<void> {\n  if (!session) {\n    return;\n  }\n  if ('end' in session && typeof session.end === 'function') {\n    await (session as {end: () => Promise<void>}).end();\n  } else {\n    session.destroy();\n  }\n}\n\nexport type PgConnectionConfig = {\n  host?: string | string[] | undefined;\n  port?: number | number[] | undefined;\n  path?: string | undefined;\n  database?: string | undefined;\n  user?: string | undefined;\n  pass?: string | null | undefined;\n  ssl?:\n    | boolean\n    | string\n    | {rejectUnauthorized?: boolean | undefined}\n    | undefined;\n};\n\nexport function extractConnectionConfig(\n  session: postgres.Sql,\n): PgConnectionConfig {\n  const opts = session.options;\n  return {\n    host: opts.host,\n    port: opts.port,\n    path: (opts as {path?: string | undefined}).path,\n    database: opts.database,\n    user: opts.user,\n    pass: opts.pass,\n    ssl:\n      typeof opts.ssl === 'object' && opts.ssl !== null\n        ? {\n            rejectUnauthorized: (\n              opts.ssl as {rejectUnauthorized?: boolean | undefined}\n            ).rejectUnauthorized,\n          }\n        : (opts.ssl as boolean | string | undefined),\n  };\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * Runs in a separate process with its own event loop and connection reconnect\n * loop to prevent slot drop races during heavy initial sync table copy or\n * SQLite index building.\n */\nexport async function keepSlotActiveUntilTakenOver(\n  lc: LogContext,\n  config: SlotKeeperConfig,\n): Promise<Readable> {\n  const {slot, lsn} = config;\n  lc.info?.(`spawning slot-keeper process for ${slot} at lsn ${lsn} ...`);\n  const worker = childWorker(SLOT_KEEPER_URL, process.env);\n  worker.onMessageType('init', () => {\n    worker.send(['start', config]);\n  });\n\n  // Wait for the slot keeper to establish replication before returning\n  await new Promise<void>((resolve, reject) => {\n    const onReady = () => {\n      cleanup();\n      resolve();\n    };\n    const onError = (err: unknown) => {\n      cleanup();\n      reject(err);\n    };\n    const onClose = (code: number) => {\n      cleanup();\n      reject(\n        new Error(\n          `slot-keeper for ${slot} exited with code ${code} before becoming ready`,\n        ),\n      );\n    };\n    function cleanup() {\n      worker.off('error', onError);\n      worker.off('close', onClose);\n    }\n    worker.onceMessageType('ready', onReady);\n    worker.once('error', onError);\n    worker.once('close', onClose);\n  });\n\n  lc.info?.(`slot-keeper active for ${slot}`);\n  return new SlotReservationHandle(worker);\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nexport function makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAsCA,SAAgB,4BACd,IACA,iBACA;CACA,OAAO,SACL,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;AACF;AAEA,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAA,GACA,kBAAkB,mBAClB,0BAIC;CACD,MAAM,UAAU,4BAA4B,IAAI,eAAe;CAQ/D,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,oBAAoB,wBAAwB;CACvE,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,uBAChD,iBAAiB,OACpC,qBAAqB,qBAAqB,IACvC,KACA,4BACN,gGAEN;CAEA,MAAM,SAAS,MAAM,qBAAqB,MAAM,IAAI,SAAS,MAAM,GAAG;CACtE,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,wBAC5C,iBAAiB,uDAE3C;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,MAAM,MAAM,mBAAmB,IAAI,QAAQ,MAAM;GACjD,IAAI,KACF,OAAO,QAAQ,KAAK,gBAAgB;GAEtC,OAAO;EACT;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;AAEA,IAAM,wCAAwC;CAC5C;CAAK;CAAK;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAK;CAAM;CAAM;CAAM;AAClE;AAEA,SAAS,yBAAyB;CAChC,OAAO,qBAAqB,eAAe,yBAAyB;EAClE,aACE;EAEF,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;;;;;;;;;;;;;;;;;;AAuBA,IAAa,uBAAb,MAAa,qBAAqB;CAChC,aAAa,MACX,IACA,SACA,MACA,KAC+B;EAC/B,IAAI;EACJ,IAAI;EACJ,IAAI;GAGF,MAAM,CAAC,OAAO,MAAM,QACjB,OACC;;;iCAGuB,IAAI,IAAI,GACjC,EACC,OAAO;GACV,aAAa,KAAK,aAAa,SAAS,IAAI,UAAU,IAAI,KAAA;GAC1D,oBAAoB,KAAK,oBACrB,SAAS,IAAI,iBAAiB,IAC9B,KAAA;EACN,SAAS,GAAG;GACV,GAAG,OAAO,+CAA+C,QAAQ,CAAC;EACpE;EACA,OAAO,IAAI,qBACT,IACA,KACA,YACA,mBACA,KAAK,IAAI,CACX;CACF;CAEA;CACA;CACA;CACA;CACA;CACA,SAAS;CACT,QAAQ;CAER,YACE,IACA,KACA,YACA,mBACA,WACA;EAGA,KAAKC,SACH,sBAAsB,KAAA,KAAa,oBAAoB,MACnD,oBACA;EACN,KAAKD,MAAM,GAAG,YAAY,eAAe,WAAW,KAAKC,MAAM,CAAC;EAChE,KAAKC,cAAc;EACnB,KAAKC,qBAAqB;EAC1B,KAAKC,aAAa;EAElB,IAAI,eAAe,KAAA,GACjB,KAAKJ,IAAI,OACP,YAAY,KAAKC,SAAS,WAAW,iCACpB,WAAW,UAAU,EAAE,4BAC1C;CAEJ;;;;;CAMA,QAAQ,CAAC,KAAK,MAAqB,KAAiC;EAClE,IAAI,KAAKI,OACP;EAEF,MAAM,QAAQ,KAAKC;EACnB,KAAKA,SAAS;EACd,IAAI,IAAI,QAAQ;OAEZ,MAAM,KAAKL,UACV,SAAS,QAAQ,KAAKE,oBAEvB;EAAA;EAGJ,KAAKE,QAAQ;EACb,MAAM,UAAU,MAAM,KAAKD;EAC3B,uBAAuB,EAAE,SAAS,OAAO;EACzC,KAAKJ,IAAI,OACP,0DACK,QAAQ,OAAO,IAAI,IAAI,MAAM,WAAW,GAAG,OAC7C,KAAKE,gBAAgB,KAAA,IAClB,MACA,KAAK,KAAKD,SAAS,KAAKC,YAAY,4BAC1C,EAAC,QAAO,CACV;EACA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,SAAgB,uBACd,oBACA,0BAMA;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAMA,kBACE,6BAA6B,KAAA,KAAa,2BAA2B,IACjE,2BACA,qBAAqB;EAE3B,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAsB,uBACpB,IACA,SACA,MACA,cACA,KACA,cAAA,GACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,KAAK,IAAI,KAAK,KAAK,GAAG,GAAI,CAAC;IACvC;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,wBAAb,cAA2C,SAAS;CAClD;CACA;CACA,WAAW;CACX;CAEA,YAAY,QAAgB;EAC1B,MAAM,EAAC,OAAO,CAAC,EAAC,CAAC;EACjB,KAAKK,UAAU;EACf,KAAKC,iBAAiB,IAAI,SAAQ,YAAW;GAC3C,OAAO,KAAK,eAAe;IACzB,aAAa,KAAKC,aAAa;IAC/B,QAAQ;GACV,CAAC;EACH,CAAC;EAED,KAAK,GAAG,eAAe,CAAC,CAAC;EACzB,OAAO,GAAG,eAAe;GACvB,IAAI,CAAC,KAAK,WACR,KAAK,QAAQ;EAEjB,CAAC;EACD,OAAO,GAAG,UAAS,QAAO;GACxB,IAAI,CAAC,KAAK,WACR,KAAK,QAAQ,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC,CAAC;EAEpE,CAAC;CACH;CAEA,SACE,OACA,UACM;EACN,IAAI,CAAC,KAAKC,UAAU;GAClB,KAAKA,WAAW;GAChB,IAAI;IACF,KAAKH,QAAQ,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;GAChC,QAAQ,CAER;GACA,KAAKE,gBAAgB,iBAAiB;IACpC,IAAI;KACF,KAAKF,QAAQ,KAAK,SAAS;IAC7B,QAAQ,CAAC;GACX,GAAG,GAAK;GACR,KAAKE,cAAc,MAAM;EAC3B;EACA,SAAS,KAAK;CAChB;CAEA,MAAM,IAAI,YAAY,KAAsB;EAC1C,KAAK,QAAQ;EAKb,IAAI,CAAC,MAJgB,QAAQ,KAAK,CAChC,KAAKD,eAAe,WAAW,IAAI,GACnC,MAAM,SAAS,EAAE,WAAW,KAAK,CACnC,CAAC,GACY;GACX,IAAI;IACF,KAAKD,QAAQ,KAAK,SAAS;GAC7B,QAAQ,CAAC;GAGT,MAAM,QAAQ,KAAK,CAAC,KAAKC,gBAAgB,MAAM,GAAK,CAAC,CAAC;EACxD;CACF;AACF;AAEA,eAAsB,WAAW,SAAmC;CAClE,IAAI,CAAC,SACH;CAEF,IAAI,SAAS,WAAW,OAAO,QAAQ,QAAQ,YAC7C,MAAO,QAAuC,IAAI;MAElD,QAAQ,QAAQ;AAEpB;AAgBA,SAAgB,wBACd,SACoB;CACpB,MAAM,OAAO,QAAQ;CACrB,OAAO;EACL,MAAM,KAAK;EACX,MAAM,KAAK;EACX,MAAO,KAAqC;EAC5C,UAAU,KAAK;EACf,MAAM,KAAK;EACX,MAAM,KAAK;EACX,KACE,OAAO,KAAK,QAAQ,YAAY,KAAK,QAAQ,OACzC,EACE,oBACE,KAAK,IACL,mBACJ,IACC,KAAK;CACd;AACF;;;;;;;;;;AAWA,eAAsB,6BACpB,IACA,QACmB;CACnB,MAAM,EAAC,MAAM,QAAO;CACpB,GAAG,OAAO,oCAAoC,KAAK,UAAU,IAAI,KAAK;CACtE,MAAM,SAAS,YAAY,iBAAiB,QAAQ,GAAG;CACvD,OAAO,cAAc,cAAc;EACjC,OAAO,KAAK,CAAC,SAAS,MAAM,CAAC;CAC/B,CAAC;CAGD,MAAM,IAAI,SAAe,SAAS,WAAW;EAC3C,MAAM,gBAAgB;GACpB,QAAQ;GACR,QAAQ;EACV;EACA,MAAM,WAAW,QAAiB;GAChC,QAAQ;GACR,OAAO,GAAG;EACZ;EACA,MAAM,WAAW,SAAiB;GAChC,QAAQ;GACR,uBACE,IAAI,MACF,mBAAmB,KAAK,oBAAoB,KAAK,uBACnD,CACF;EACF;EACA,SAAS,UAAU;GACjB,OAAO,IAAI,SAAS,OAAO;GAC3B,OAAO,IAAI,SAAS,OAAO;EAC7B;EACA,OAAO,gBAAgB,SAAS,OAAO;EACvC,OAAO,KAAK,SAAS,OAAO;EAC5B,OAAO,KAAK,SAAS,OAAO;CAC9B,CAAC;CAED,GAAG,OAAO,0BAA0B,MAAM;CAC1C,OAAO,IAAI,sBAAsB,MAAM;AACzC;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAgB,QAAQ,KAAqB;CAC3C,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}

package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts

@@ -0,0 +1,16 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { PostgresDB } from '../../../types/pg.ts';
+import type { ShardID } from '../../../types/shards.ts';
+type Options = {
+    inactiveSlotCleanupTimeoutMs?: number | undefined;
+    pollIntervalMs?: number | undefined;
+};
+export declare class ReplicationSlotCleanupMonitor {
+    #private;
+    constructor(lc: LogContext, shard: ShardID, upstream: PostgresDB, upstreamPgServerVersionNum: number, currentSlot: string, { inactiveSlotCleanupTimeoutMs, pollIntervalMs }?: Options);
+    start(): void;
+    stop(): void;
+    getSlotsToCleanup(now: Date): Promise<string[]>;
+}
+export {};
+//# sourceMappingURL=replication-slot-cleanup-monitor.d.ts.map

package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replication-slot-cleanup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAItD,KAAK,OAAO,GAAG;IACb,4BAA4B,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClD,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAKF,qBAAa,6BAA6B;;IAkCxC,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,QAAQ,EAAE,UAAU,EACpB,0BAA0B,EAAE,MAAM,EAClC,WAAW,EAAE,MAAM,EACnB,EAAC,4BAA4B,EAAE,cAAc,EAAC,GAAE,OAAY,EAW7D;IAED,KAAK,SAYJ;IAED,IAAI,SAIH;IAiBK,iBAAiB,CAAC,GAAG,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAqEpD;CACF"}

package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js

@@ -0,0 +1,113 @@
+import { replicationSlotPrefix } from "./schema/shard.js";
+import "../../../types/pg-versions.js";
+import { dropInactiveSlotsAndReplicas } from "./replication-slots.js";
+//#region ../zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.ts
+var DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS = 300 * 1e3;
+var DEFAULT_POLL_INTERVAL_MS = 60 * 1e3;
+var ReplicationSlotCleanupMonitor = class {
+	#lc;
+	/** The shard constrains which slots are monitored (e.g. zero_0_*). */
+	#shard;
+	#upstream;
+	/**
+	* This slot that this task is using. Cleanup of slots is predicated on
+	* this slot remaining active, in order to guarantee at least one healthy
+	* slot in the shard.
+	*/
+	#currentSlot;
+	/**
+	* How long a slot is inactive (while the currentSlot is active) before it
+	* is eligible to be cleaned up. Because new slots are immediately connected
+	* to, even while an initial-sync or replica-restore is in progress, slots
+	* should only be inactive if no server is using it, or if a server was
+	* disconnected (e.g. a pg restart) and is in the process of reconnecting
+	* to it (see `keepSlotActiveUntilTakenOver`).
+	*
+	* As such, this timeout should be long enough to account for a server
+	* reconnecting to upstream. Note again that the cleanup monitor only runs
+	* while at least one replication slot is active, guaranteeing that upstream
+	* replication is working.
+	*/
+	#inactiveSlotCleanupTimeoutMs;
+	#pollIntervalMs;
+	/** Manually tracked inactive times for PG < 17. null for PG 17+ */
+	#firstInactiveTimes;
+	#pollTimer;
+	constructor(lc, shard, upstream, upstreamPgServerVersionNum, currentSlot, { inactiveSlotCleanupTimeoutMs, pollIntervalMs } = {}) {
+		this.#lc = lc.withContext("component", "replication-slot-cleanup-monitor");
+		this.#shard = shard;
+		this.#upstream = upstream;
+		this.#currentSlot = currentSlot;
+		this.#inactiveSlotCleanupTimeoutMs = inactiveSlotCleanupTimeoutMs ?? DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS;
+		this.#pollIntervalMs = pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+		this.#firstInactiveTimes = upstreamPgServerVersionNum >= 17e4 ? null : /* @__PURE__ */ new Map();
+	}
+	start() {
+		if (this.#pollTimer) {
+			this.#lc.warn?.(`monitor already started`);
+			return;
+		}
+		this.#pollTimer = setInterval(() => this.#poll(/* @__PURE__ */ new Date()).catch((e) => this.#lc.warn?.(`error checking replication slots`, e)), this.#pollIntervalMs);
+	}
+	stop() {
+		clearInterval(this.#pollTimer);
+		this.#pollTimer = void 0;
+		this.#firstInactiveTimes?.clear();
+	}
+	async #poll(now) {
+		if (!this.#pollTimer) return;
+		const slots = await this.getSlotsToCleanup(now);
+		if (slots.length) await dropInactiveSlotsAndReplicas(this.#lc, this.#upstream, this.#shard, slots);
+	}
+	async getSlotsToCleanup(now) {
+		const slotPoolPrefix = replicationSlotPrefix(this.#shard);
+		const slotsToCleanUp = [];
+		const trackedInactiveTimes = this.#firstInactiveTimes;
+		if (trackedInactiveTimes) {
+			const shardSlots = await this.#upstream`
+        SELECT slot_name AS slot, active FROM pg_replication_slots
+          WHERE slot_name LIKE ${slotPoolPrefix + "%"};
+      `;
+			for (const { slot, active } of shardSlots) {
+				if (slot === this.#currentSlot && !active) {
+					this.#lc.warn?.(`current slot ${this.#currentSlot} is not active. disabling cleanup.`);
+					this.stop();
+					return [];
+				}
+				if (active) trackedInactiveTimes.delete(slot);
+				else {
+					const inactiveSince = trackedInactiveTimes.get(slot);
+					if (!inactiveSince) trackedInactiveTimes.set(slot, now);
+					else if (now.getTime() - inactiveSince.getTime() >= this.#inactiveSlotCleanupTimeoutMs) {
+						this.#lc.info?.(`slot ${slot} has been inactive since ${inactiveSince.toISOString()} and is eligible for cleanup`);
+						slotsToCleanUp.push(slot);
+					}
+				}
+			}
+		} else {
+			const expiration = now.getTime() - this.#inactiveSlotCleanupTimeoutMs;
+			const inactiveSlots = await this.#upstream`
+        SELECT slot_name AS slot, inactive_since AS "inactiveSince" FROM pg_replication_slots
+          WHERE slot_name LIKE ${slotPoolPrefix + "%"}
+            AND active = false
+            AND inactive_since IS NOT NULL;
+      `;
+			for (const { slot, inactiveSince } of inactiveSlots) {
+				if (slot === this.#currentSlot) {
+					this.#lc.warn?.(`current slot ${this.#currentSlot} is not active. disabling cleanup.`);
+					this.stop();
+					return [];
+				}
+				if (inactiveSince <= expiration) {
+					this.#lc.info?.(`slot ${slot} has been inactive since ${new Date(inactiveSince).toISOString()} and is eligible for cleanup`);
+					slotsToCleanUp.push(slot);
+				}
+			}
+		}
+		return slotsToCleanUp;
+	}
+};
+//#endregion
+export { ReplicationSlotCleanupMonitor };
+
+//# sourceMappingURL=replication-slot-cleanup-monitor.js.map

package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"replication-slot-cleanup-monitor.js","names":["#lc","#shard","#upstream","#currentSlot","#inactiveSlotCleanupTimeoutMs","#pollIntervalMs","#firstInactiveTimes","#pollTimer","#poll"],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport type {PostgresDB} from '../../../types/pg.ts';\nimport type {ShardID} from '../../../types/shards.ts';\nimport {dropInactiveSlotsAndReplicas} from './replication-slots.ts';\nimport {replicationSlotPrefix} from './schema/shard.ts';\n\ntype Options = {\n  inactiveSlotCleanupTimeoutMs?: number | undefined;\n  pollIntervalMs?: number | undefined;\n};\n\nconst DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS = 5 * 60 * 1000;\nconst DEFAULT_POLL_INTERVAL_MS = 60 * 1000;\n\nexport class ReplicationSlotCleanupMonitor {\n  readonly #lc: LogContext;\n\n  /** The shard constrains which slots are monitored (e.g. zero_0_*). */\n  readonly #shard: ShardID;\n  readonly #upstream: PostgresDB;\n  /**\n   * This slot that this task is using. Cleanup of slots is predicated on\n   * this slot remaining active, in order to guarantee at least one healthy\n   * slot in the shard.\n   */\n  readonly #currentSlot: string;\n\n  /**\n   * How long a slot is inactive (while the currentSlot is active) before it\n   * is eligible to be cleaned up. Because new slots are immediately connected\n   * to, even while an initial-sync or replica-restore is in progress, slots\n   * should only be inactive if no server is using it, or if a server was\n   * disconnected (e.g. a pg restart) and is in the process of reconnecting\n   * to it (see `keepSlotActiveUntilTakenOver`).\n   *\n   * As such, this timeout should be long enough to account for a server\n   * reconnecting to upstream. Note again that the cleanup monitor only runs\n   * while at least one replication slot is active, guaranteeing that upstream\n   * replication is working.\n   */\n  readonly #inactiveSlotCleanupTimeoutMs: number;\n  readonly #pollIntervalMs: number;\n\n  /** Manually tracked inactive times for PG < 17. null for PG 17+ */\n  readonly #firstInactiveTimes: Map<string, Date> | null;\n\n  #pollTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    upstream: PostgresDB,\n    upstreamPgServerVersionNum: number,\n    currentSlot: string,\n    {inactiveSlotCleanupTimeoutMs, pollIntervalMs}: Options = {},\n  ) {\n    this.#lc = lc.withContext('component', 'replication-slot-cleanup-monitor');\n    this.#shard = shard;\n    this.#upstream = upstream;\n    this.#currentSlot = currentSlot;\n    this.#inactiveSlotCleanupTimeoutMs =\n      inactiveSlotCleanupTimeoutMs ?? DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS;\n    this.#pollIntervalMs = pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;\n    this.#firstInactiveTimes =\n      upstreamPgServerVersionNum >= PG_17 ? null : new Map();\n  }\n\n  start() {\n    if (this.#pollTimer) {\n      this.#lc.warn?.(`monitor already started`);\n      return;\n    }\n    this.#pollTimer = setInterval(\n      () =>\n        this.#poll(new Date()).catch(e =>\n          this.#lc.warn?.(`error checking replication slots`, e),\n        ),\n      this.#pollIntervalMs,\n    );\n  }\n\n  stop() {\n    clearInterval(this.#pollTimer);\n    this.#pollTimer = undefined;\n    this.#firstInactiveTimes?.clear();\n  }\n\n  async #poll(now: Date) {\n    if (!this.#pollTimer) {\n      return; // concurrently stopped\n    }\n    const slots = await this.getSlotsToCleanup(now);\n    if (slots.length) {\n      await dropInactiveSlotsAndReplicas(\n        this.#lc,\n        this.#upstream,\n        this.#shard,\n        slots,\n      );\n    }\n  }\n\n  async getSlotsToCleanup(now: Date): Promise<string[]> {\n    const slotPoolPrefix = replicationSlotPrefix(this.#shard);\n\n    const slotsToCleanUp: string[] = [];\n    const trackedInactiveTimes = this.#firstInactiveTimes;\n\n    if (trackedInactiveTimes) {\n      // For PG <17, each poll tracks and updates earliest time the slot was\n      // observed to be inactive.\n      const shardSlots = await this.#upstream<{slot: string; active: boolean}[]>\n      /*sql*/ `\n        SELECT slot_name AS slot, active FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n      `;\n      for (const {slot, active} of shardSlots) {\n        if (slot === this.#currentSlot && !active) {\n          this.#lc.warn?.(\n            `current slot ${this.#currentSlot} is not active. disabling cleanup.`,\n          );\n          this.stop();\n          return [];\n        }\n        if (active) {\n          trackedInactiveTimes.delete(slot);\n        } else {\n          const inactiveSince = trackedInactiveTimes.get(slot);\n          if (!inactiveSince) {\n            trackedInactiveTimes.set(slot, now);\n          } else if (\n            now.getTime() - inactiveSince.getTime() >=\n            this.#inactiveSlotCleanupTimeoutMs\n          ) {\n            this.#lc.info?.(\n              `slot ${slot} has been inactive since ${inactiveSince.toISOString()} and is eligible for cleanup`,\n            );\n            slotsToCleanUp.push(slot);\n          }\n        }\n      }\n    } else {\n      // PG 17+ is simpler as inactivity time is tracked in the table.\n      const expiration = now.getTime() - this.#inactiveSlotCleanupTimeoutMs;\n      const inactiveSlots = await this.#upstream<\n        {slot: string; inactiveSince: number}[]\n      > /*sql*/ `\n        SELECT slot_name AS slot, inactive_since AS \"inactiveSince\" FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'}\n            AND active = false\n            AND inactive_since IS NOT NULL;\n      `;\n      for (const {slot, inactiveSince} of inactiveSlots) {\n        if (slot === this.#currentSlot) {\n          this.#lc.warn?.(\n            `current slot ${this.#currentSlot} is not active. disabling cleanup.`,\n          );\n          this.stop();\n          return [];\n        }\n        // The slot has been inactive since at or before the expiration cutoff\n        // (now - timeout), i.e. inactive for at least the timeout duration.\n        if (inactiveSince <= expiration) {\n          this.#lc.info?.(\n            `slot ${slot} has been inactive since ${new Date(inactiveSince).toISOString()} and is eligible for cleanup`,\n          );\n          slotsToCleanUp.push(slot);\n        }\n      }\n    }\n    return slotsToCleanUp;\n  }\n}\n"],"mappings":";;;;AAYA,IAAM,2CAA2C,MAAS;AAC1D,IAAM,2BAA2B,KAAK;AAEtC,IAAa,gCAAb,MAA2C;CACzC;;CAGA;CACA;;;;;;CAMA;;;;;;;;;;;;;;CAeA;CACA;;CAGA;CAEA;CAEA,YACE,IACA,OACA,UACA,4BACA,aACA,EAAC,8BAA8B,mBAA2B,CAAC,GAC3D;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kCAAkC;EACzE,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,eAAe;EACpB,KAAKC,gCACH,gCAAgC;EAClC,KAAKC,kBAAkB,kBAAkB;EACzC,KAAKC,sBACH,8BAAA,OAAsC,uBAAO,IAAI,IAAI;CACzD;CAEA,QAAQ;EACN,IAAI,KAAKC,YAAY;GACnB,KAAKP,IAAI,OAAO,yBAAyB;GACzC;EACF;EACA,KAAKO,aAAa,kBAEd,KAAKC,sBAAM,IAAI,KAAK,CAAC,EAAE,OAAM,MAC3B,KAAKR,IAAI,OAAO,oCAAoC,CAAC,CACvD,GACF,KAAKK,eACP;CACF;CAEA,OAAO;EACL,cAAc,KAAKE,UAAU;EAC7B,KAAKA,aAAa,KAAA;EAClB,KAAKD,qBAAqB,MAAM;CAClC;CAEA,MAAME,MAAM,KAAW;EACrB,IAAI,CAAC,KAAKD,YACR;EAEF,MAAM,QAAQ,MAAM,KAAK,kBAAkB,GAAG;EAC9C,IAAI,MAAM,QACR,MAAM,6BACJ,KAAKP,KACL,KAAKE,WACL,KAAKD,QACL,KACF;CAEJ;CAEA,MAAM,kBAAkB,KAA8B;EACpD,MAAM,iBAAiB,sBAAsB,KAAKA,MAAM;EAExD,MAAM,iBAA2B,CAAC;EAClC,MAAM,uBAAuB,KAAKK;EAElC,IAAI,sBAAsB;GAGxB,MAAM,aAAa,MAAM,KAAKJ,SACtB;;iCAEmB,iBAAiB,IAAI;;GAEhD,KAAK,MAAM,EAAC,MAAM,YAAW,YAAY;IACvC,IAAI,SAAS,KAAKC,gBAAgB,CAAC,QAAQ;KACzC,KAAKH,IAAI,OACP,gBAAgB,KAAKG,aAAa,mCACpC;KACA,KAAK,KAAK;KACV,OAAO,CAAC;IACV;IACA,IAAI,QACF,qBAAqB,OAAO,IAAI;SAC3B;KACL,MAAM,gBAAgB,qBAAqB,IAAI,IAAI;KACnD,IAAI,CAAC,eACH,qBAAqB,IAAI,MAAM,GAAG;UAC7B,IACL,IAAI,QAAQ,IAAI,cAAc,QAAQ,KACtC,KAAKC,+BACL;MACA,KAAKJ,IAAI,OACP,QAAQ,KAAK,2BAA2B,cAAc,YAAY,EAAE,6BACtE;MACA,eAAe,KAAK,IAAI;KAC1B;IACF;GACF;EACF,OAAO;GAEL,MAAM,aAAa,IAAI,QAAQ,IAAI,KAAKI;GACxC,MAAM,gBAAgB,MAAM,KAAKF,SAEvB;;iCAEiB,iBAAiB,IAAI;;;;GAIhD,KAAK,MAAM,EAAC,MAAM,mBAAkB,eAAe;IACjD,IAAI,SAAS,KAAKC,cAAc;KAC9B,KAAKH,IAAI,OACP,gBAAgB,KAAKG,aAAa,mCACpC;KACA,KAAK,KAAK;KACV,OAAO,CAAC;IACV;IAGA,IAAI,iBAAiB,YAAY;KAC/B,KAAKH,IAAI,OACP,QAAQ,KAAK,2BAA2B,IAAI,KAAK,aAAa,EAAE,YAAY,EAAE,6BAChF;KACA,eAAe,KAAK,IAAI;IAC1B;GACF;EACF;EACA,OAAO;CACT;AACF"}