@rocicorp/zero 1.10.0-canary.2 → 1.10.0-canary.21

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js

@@ -12,8 +12,8 @@ import postgres from "postgres";
 import { PG_ADMIN_SHUTDOWN, PG_OBJECT_IN_USE, PG_OBJECT_NOT_IN_PREREQUISITE_STATE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/logical-replication/stream.ts
 var DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;
-async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE, applicationName = "zero-replicator", inboundTimeoutOverrideMs) {
-	const session = postgres(defu({
+function createReplicationSessionFor(db, applicationName) {
+	return postgres(defu({
 		max: 1,
 		["fetch_types"]: false,
 		["idle_timeout"]: null,
@@ -23,6 +23,9 @@ async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSl
 			replication: "database"
 		}
 	}, db.options));
+}
+async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE, applicationName = "zero-replicator", inboundTimeoutOverrideMs) {
+	const session = createReplicationSessionFor(db, applicationName);
 	const [{ walSenderTimeoutMs }] = await session`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 
         AS "walSenderTimeoutMs" FROM pg_settings
         WHERE name = 'wal_sender_timeout'`.simple();
@@ -184,6 +187,43 @@ async function startReplicationStream(lc, session, slot, publications, lsn, maxA
 	}
 	throw new Error(`exceeded max attempts (${maxAttempts}) to start the Postgres stream`);
 }
+/**
+* Starts a "dummy" replication session for the specified `slot` which:
+* * does not consume any records
+* * keeps the session alive by sending status messages at the starting LSN
+*
+* This essentially reserves the replication slot for this process by marking
+* it as "active" in pg_replication_slots, allowing replication slot cleanup
+* logic to distinguish a new-but-initializing replication slot from one that
+* has been abandoned / orphaned.
+*
+* When initialization (e.g. initial-sync) has finished, the server should take
+* over the replication slot using pg_terminate_backend, which will close the
+* session.
+*
+* Note that caller should not close the supplied `session`, as that will
+* also terminate the dummy replication session.
+*/
+async function keepSlotActiveUntilTakenOver(lc, session, slot, dummyPublication, lsn) {
+	const [readable, writeable] = await startReplicationStream(lc, session, slot, [dummyPublication], lsn, 1);
+	lc.info?.(`keeping ${slot} active until taken over ...`);
+	const keepalive = setInterval(() => writeable.write(makeAck(lsn)), 1e4);
+	readable.on("close", async () => {
+		lc.info?.(`released initial reservation of ${slot}`);
+		clearInterval(keepalive);
+		try {
+			await session.end();
+			lc.info?.(`ended session`);
+		} catch (e) {
+			lc.warn?.(`error ending session`, e);
+		}
+	});
+	readable.once("error", (e) => {
+		lc.info?.(`closing replication session to ${session.options.host}`, e);
+		readable.destroy();
+	});
+	return readable;
+}
 function parseStreamMessage(lc, buffer, parser) {
 	if (buffer[0] !== 119 && buffer[0] !== 107) {
 		lc.warn?.("Unknown message", buffer[0]);
@@ -207,6 +247,6 @@ function makeAck(lsn) {
 	return x;
 }
 //#endregion
-export { subscribe };
+export { createReplicationSessionFor, keepSlotActiveUntilTakenOver, subscribe };
 
 //# sourceMappingURL=stream.js.map

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n  inboundTimeoutOverrideMs?: number | undefined,\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. ` +\n          `Inbound timeout: ${inboundTimeoutMs}ms` +\n          (inboundTimeoutMs === walSenderTimeoutMs * 2\n            ? ''\n            : ' (configured override)')\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> inbound timeout ${inboundTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      return parseStreamMessage(lc, buffer, parser);\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * An `inboundTimeoutOverrideMs` (when positive) replaces the default\n * `2x wal_sender_timeout` inbound threshold. The two timeouts protect\n * different parties with asymmetric false-positive costs: `wal_sender_timeout`\n * is the server reaping dead clients (a false positive there costs a cheap\n * reconnect), while the inbound watchdog tears down the stream and replays the\n * in-flight transaction — expensive, and self-defeating when the wal sender is\n * legitimately silent for long stretches (e.g. decoding through WAL from\n * unpublished tables, or assembling a large transaction that pgoutput only\n * sends at commit). Operators who don't own the server setting (e.g. managed\n * or operator-provisioned Postgres with an aggressive cluster-wide\n * `wal_sender_timeout`) can widen the client-side fuse independently.\n *\n * The override has no effect when `wal_sender_timeout` is disabled (0): the\n * server then sends no periodic keepalives, so inbound silence is normal even\n * on an idle healthy connection and any silence-based watchdog would\n * false-positive.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(\n  walSenderTimeoutMs: number,\n  inboundTimeoutOverrideMs?: number | undefined,\n): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.). A 0 / negative / NaN override\n    // falls back to the default rather than degenerating the threshold.\n    inboundTimeoutMs:\n      inboundTimeoutOverrideMs !== undefined && inboundTimeoutOverrideMs > 0\n        ? inboundTimeoutOverrideMs\n        : walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;AAmBA,IAAM,6CAA6C;AAcnD,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAClB,0BAIC;CACD,MAAM,UAAU,SACd,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;CAQA,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,oBAAoB,wBAAwB;CACvE,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,uBAChD,iBAAiB,OACpC,qBAAqB,qBAAqB,IACvC,KACA,4BACN,gGAEN;CAEA,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,wBAC5C,iBAAiB,uDAE3C;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,OAAO,mBAAmB,IAAI,QAAQ,MAAM;EAC9C;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,SAAgB,uBACd,oBACA,0BAMA;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAMA,kBACE,6BAA6B,KAAA,KAAa,2BAA2B,IACjE,2BACA,qBAAqB;EAE3B,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}
+{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport function createReplicationSessionFor(\n  db: PostgresDB,\n  applicationName: string,\n) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n  inboundTimeoutOverrideMs?: number | undefined,\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = createReplicationSessionFor(db, applicationName);\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. ` +\n          `Inbound timeout: ${inboundTimeoutMs}ms` +\n          (inboundTimeoutMs === walSenderTimeoutMs * 2\n            ? ''\n            : ' (configured override)')\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> inbound timeout ${inboundTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      return parseStreamMessage(lc, buffer, parser);\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * An `inboundTimeoutOverrideMs` (when positive) replaces the default\n * `2x wal_sender_timeout` inbound threshold. The two timeouts protect\n * different parties with asymmetric false-positive costs: `wal_sender_timeout`\n * is the server reaping dead clients (a false positive there costs a cheap\n * reconnect), while the inbound watchdog tears down the stream and replays the\n * in-flight transaction — expensive, and self-defeating when the wal sender is\n * legitimately silent for long stretches (e.g. decoding through WAL from\n * unpublished tables, or assembling a large transaction that pgoutput only\n * sends at commit). Operators who don't own the server setting (e.g. managed\n * or operator-provisioned Postgres with an aggressive cluster-wide\n * `wal_sender_timeout`) can widen the client-side fuse independently.\n *\n * The override has no effect when `wal_sender_timeout` is disabled (0): the\n * server then sends no periodic keepalives, so inbound silence is normal even\n * on an idle healthy connection and any silence-based watchdog would\n * false-positive.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(\n  walSenderTimeoutMs: number,\n  inboundTimeoutOverrideMs?: number | undefined,\n): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.). A 0 / negative / NaN override\n    // falls back to the default rather than degenerating the threshold.\n    inboundTimeoutMs:\n      inboundTimeoutOverrideMs !== undefined && inboundTimeoutOverrideMs > 0\n        ? inboundTimeoutOverrideMs\n        : walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * This essentially reserves the replication slot for this process by marking\n * it as \"active\" in pg_replication_slots, allowing replication slot cleanup\n * logic to distinguish a new-but-initializing replication slot from one that\n * has been abandoned / orphaned.\n *\n * When initialization (e.g. initial-sync) has finished, the server should take\n * over the replication slot using pg_terminate_backend, which will close the\n * session.\n *\n * Note that caller should not close the supplied `session`, as that will\n * also terminate the dummy replication session.\n */\nexport async function keepSlotActiveUntilTakenOver(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  dummyPublication: string,\n  lsn: bigint,\n) {\n  const [readable, writeable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    [dummyPublication],\n    lsn,\n    1,\n  );\n  lc.info?.(`keeping ${slot} active until taken over ...`);\n  // Send periodic status messages upstream to indicate that the session is\n  // alive, but keep the confirmed_flush_lsn at the initial lsn.\n  const keepalive = setInterval(() => writeable.write(makeAck(lsn)), 10_000);\n\n  readable.on('close', async () => {\n    lc.info?.(`released initial reservation of ${slot}`);\n    clearInterval(keepalive);\n    try {\n      await session.end();\n      lc.info?.(`ended session`);\n    } catch (e) {\n      lc.warn?.(`error ending session`, e);\n    }\n  });\n  readable.once('error', e => {\n    lc.info?.(`closing replication session to ${session.options.host}`, e);\n    readable.destroy();\n  });\n  return readable;\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;AAmBA,IAAM,6CAA6C;AAcnD,SAAgB,4BACd,IACA,iBACA;CACA,OAAO,SACL,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;AACF;AAEA,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAClB,0BAIC;CACD,MAAM,UAAU,4BAA4B,IAAI,eAAe;CAQ/D,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,oBAAoB,wBAAwB;CACvE,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,uBAChD,iBAAiB,OACpC,qBAAqB,qBAAqB,IACvC,KACA,4BACN,gGAEN;CAEA,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,wBAC5C,iBAAiB,uDAE3C;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,OAAO,mBAAmB,IAAI,QAAQ,MAAM;EAC9C;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,SAAgB,uBACd,oBACA,0BAMA;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAMA,kBACE,6BAA6B,KAAA,KAAa,2BAA2B,IACjE,2BACA,qBAAqB;EAE3B,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,SACA,MACA,kBACA,KACA;CACA,MAAM,CAAC,UAAU,aAAa,MAAM,uBAClC,IACA,SACA,MACA,CAAC,gBAAgB,GACjB,KACA,CACF;CACA,GAAG,OAAO,WAAW,KAAK,6BAA6B;CAGvD,MAAM,YAAY,kBAAkB,UAAU,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAM;CAEzE,SAAS,GAAG,SAAS,YAAY;EAC/B,GAAG,OAAO,mCAAmC,MAAM;EACnD,cAAc,SAAS;EACvB,IAAI;GACF,MAAM,QAAQ,IAAI;GAClB,GAAG,OAAO,eAAe;EAC3B,SAAS,GAAG;GACV,GAAG,OAAO,wBAAwB,CAAC;EACrC;CACF,CAAC;CACD,SAAS,KAAK,UAAS,MAAK;EAC1B,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,QAAQ,CAAC;EACrE,SAAS,QAAQ;CACnB,CAAC;CACD,OAAO;AACT;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}

package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts

@@ -1,3 +1,4 @@
+import type { Readable } from 'node:stream';
 import type { LogContext } from '@rocicorp/logger';
 import type postgres from 'postgres';
 import { type PostgresDB } from '../../../types/pg.ts';
@@ -9,6 +10,11 @@ export type ReplicationSlot = {
     snapshot_name: string;
     output_plugin: string;
 };
+export type ReplicationSlotResult<T> = {
+    slot: ReplicationSlot;
+    capturedSnapshot: T;
+    initialSession: Readable;
+};
 export type CreateSlotSpec = {
     slotName: string;
     failover?: boolean;
@@ -38,8 +44,25 @@ export declare function createReplicationSlot(lc: LogContext, session: postgres.
  * 2. Running replication managers (which use an earlier replica of a lower
  *    rank) will not delete the new slot during their cleanup logic, since
  *    the slot will belong to a replica of a higher rank.
+ *
+ * When the replication slot is created, `captureSnapshot` callback is first
+ * run while the snapshot at the consistent point is available. The callback
+ * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)
+ * in order to use it. Once the callback completes, a placeholder replication
+ * session (i.e. that does not consume any messages) is started, in order to
+ * reserve the slot by mark it "active", effectively distinguishing the
+ * initializing session from a failed or abandoned session.
+ *
+ * Note that starting the replication session releases the initial snapshot,
+ * which is why the callback must capture it synchronously, before the
+ * replication session is started.
+ *
+ * The placeholder replication session is closed when the
+ * PostgresChangeSource takes over the slot to process the stream in earnest.
+ * It is also returned for cleanup in the case of failures (and for control
+ * in unit tests).
  */
-export declare function createReplicaAndSlot(lc: LogContext, sql: PostgresDB, replicationSession: postgres.Sql, shard: ShardID, replicaID: string, failover: boolean, backupOptions: BackupOptions): Promise<ReplicationSlot>;
+export declare function createReplicaAndSlot<T>(lc: LogContext, sql: PostgresDB, sessionName: string, shard: ShardID, replicaID: string, failover: boolean, backupOptions: BackupOptions, captureSnapshot: (snapshot: string) => Promise<T>): Promise<ReplicationSlotResult<T>>;
 /**
  * Deletes "old" replicas (i.e. those with a lower rank than the current)
  * and attempts to drop replication slots that are not associated with any

package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAGtE,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IAGjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAInB,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAoBF,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,EACE,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,WAAoC,GACrC,EAAE,cAAc,GAChB,OAAO,CAAC,eAAe,CAAC,CAwC1B;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,oBAAoB,CACxC,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,kBAAkB,EAAE,QAAQ,CAAC,GAAG,EAChC,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,eAAe,CAAC,CAoE1B;AAED;;;;;;;;;GASG;AACH,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC,CAY9D;AA+DD,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,UAUvC"}
+{"version":3,"file":"replication-slots.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAK1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAOtE,OAAO,EAKL,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,qBAAqB,CAAC,CAAC,IAAI;IACrC,IAAI,EAAE,eAAe,CAAC;IACtB,gBAAgB,EAAE,CAAC,CAAC;IACpB,cAAc,EAAE,QAAQ,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IAGjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAInB,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAoBF,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,EACE,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,WAAoC,GACrC,EAAE,cAAc,GAChB,OAAO,CAAC,eAAe,CAAC,CAwC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GAChD,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAuFnC;AAED;;;;;;;;;GASG;AACH,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC,CAY9D;AA+DD,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,UAUvC"}

package/out/zero-cache/src/services/change-source/pg/replication-slots.js

@@ -1,9 +1,10 @@
 import { upstreamSchema } from "../../../types/shards.js";
 import { isPostgresError } from "../../../types/pg.js";
-import { toStateVersionString } from "./lsn.js";
+import { toBigInt, toStateVersionString } from "./lsn.js";
 import { runTx } from "../../../db/run-transaction.js";
-import { createReplica, replicationSlotExpression, replicationSlotPrefix } from "./schema/shard.js";
+import { createReplica, metadataPublicationName, replicationSlotExpression, replicationSlotPrefix } from "./schema/shard.js";
 import { orTimeout } from "../../../types/timeout.js";
+import { createReplicationSessionFor, keepSlotActiveUntilTakenOver } from "./logical-replication/stream.js";
 import { PG_CONFIGURATION_LIMIT_EXCEEDED, PG_INSUFFICIENT_PRIVILEGE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/replication-slots.ts
 var CREATE_REPLICATION_SLOT_TIMEOUT_MS = 3e4;
@@ -44,16 +45,34 @@ async function createReplicationSlot(lc, session, { slotName, failover, temporar
 * 2. Running replication managers (which use an earlier replica of a lower
 *    rank) will not delete the new slot during their cleanup logic, since
 *    the slot will belong to a replica of a higher rank.
+*
+* When the replication slot is created, `captureSnapshot` callback is first
+* run while the snapshot at the consistent point is available. The callback
+* must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)
+* in order to use it. Once the callback completes, a placeholder replication
+* session (i.e. that does not consume any messages) is started, in order to
+* reserve the slot by mark it "active", effectively distinguishing the
+* initializing session from a failed or abandoned session.
+*
+* Note that starting the replication session releases the initial snapshot,
+* which is why the callback must capture it synchronously, before the
+* replication session is started.
+*
+* The placeholder replication session is closed when the
+* PostgresChangeSource takes over the slot to process the stream in earnest.
+* It is also returned for cleanup in the case of failures (and for control
+* in unit tests).
 */
-async function createReplicaAndSlot(lc, sql, replicationSession, shard, replicaID, failover, backupOptions) {
+async function createReplicaAndSlot(lc, sql, sessionName, shard, replicaID, failover, backupOptions, captureSnapshot) {
+	const replicationSession = createReplicationSessionFor(sql, sessionName);
 	const lockName = replicationSlotManagementLock(shard);
 	const slotPoolPrefix = replicationSlotPrefix(shard);
 	for (let first = true;; first = false) {
 		await dropUnclaimedSlots(lc, sql, shard);
+		let slotName;
 		try {
 			return await runTx(sql, async (tx) => {
 				await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;
-				let slotName;
 				const names = await tx`
           SELECT slot_name as name FROM pg_replication_slots
             WHERE slot_name LIKE ${slotPoolPrefix + "%"};
@@ -70,8 +89,14 @@ async function createReplicaAndSlot(lc, sql, replicationSession, shard, replicaI
 					slotName,
 					failover
 				});
+				const capturedSnapshot = await captureSnapshot(slot.snapshot_name);
+				const initialSession = await keepSlotActiveUntilTakenOver(lc, replicationSession, slot.slot_name, metadataPublicationName(shard.appID, shard.shardNum), toBigInt(slot.consistent_point));
 				await createReplica(tx, shard, replicaID, slot.slot_name, toStateVersionString(slot.consistent_point), backupOptions);
-				return slot;
+				return {
+					slot,
+					capturedSnapshot,
+					initialSession
+				};
 			});
 		} catch (e) {
 			if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {
@@ -86,6 +111,11 @@ async function createReplicaAndSlot(lc, sql, replicationSession, shard, replicaI
             WHERE replicas.slot = slots.slot_name AND NOT slots.active`;
 				continue;
 			}
+			if (slotName) {
+				lc.warn?.(`deleting slot ${slotName} due to error`, e);
+				await replicationSession.end();
+				await sql`SELECT pg_drop_replication_slot(${slotName})`;
+			}
 			throw e;
 		}
 	}

package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type postgres from 'postgres';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {\n  createReplica,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: must be false if pgVersion < PG_17. Caller must verify.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nconst CREATE_REPLICATION_SLOT_TIMEOUT_MS = 30_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n  if (raced === 'timed-out') {\n    // Create slot can block indefinitely waiting for old transactions. End\n    // this connection in the background and fail fast so the process restarts.\n    void session\n      .end()\n      .catch(e =>\n        lc.warn?.(`Error closing timed out replication slot session`, e),\n      );\n    throw new Error(\n      `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}. ` +\n        `Crashing to force a clean restart.`,\n    );\n  }\n  const [slot] = raced;\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n */\nexport async function createReplicaAndSlot(\n  lc: LogContext,\n  sql: PostgresDB,\n  replicationSession: postgres.Sql,\n  shard: ShardID,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n): Promise<ReplicationSlot> {\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n  for (let first = true; ; first = false) {\n    await dropUnclaimedSlots(lc, sql, shard);\n    try {\n      return await runTx(sql, async tx => {\n        await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n        // Pick an available slotName from the slotPoolPrefix pool.\n        let slotName: string;\n        const names = await tx<{name: string}[]> /*sql*/ `\n          SELECT slot_name as name FROM pg_replication_slots\n            WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n        `.values();\n        const inUse = new Set(names.flat());\n        for (let next = 0; ; next++) {\n          const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n          if (!inUse.has(candidateName)) {\n            slotName = candidateName;\n            break;\n          }\n        }\n\n        const slot = await createReplicationSlot(lc, replicationSession, {\n          slotName,\n          failover,\n        });\n\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          toStateVersionString(slot.consistent_point),\n          backupOptions,\n        );\n\n        return slot;\n      });\n    } catch (e) {\n      if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Deletes \"old\" replicas (i.e. those with a lower rank than the current)\n * and attempts to drop replication slots that are not associated with any\n * replica.\n *\n * If a slot could not be dropped because there is still an active subscriber,\n * it will be reflected in the `draining` count that is returned. When there\n * are draining slots, the method should be retried until all orphaned slots\n * have been dropped.\n */\nexport async function dropOldReplicasAndSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  beforeRank: bigint,\n): Promise<{dropped: number; active: number; draining: number}> {\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n  const oldReplicas = await sql`\n    SELECT id, rank::float8, slot, version, \"initialSyncContext\", \"subscriberContext\"\n     FROM ${sql(replicasTable)} WHERE rank < ${beforeRank};\n  `;\n  if (oldReplicas.length) {\n    lc.info?.(`Deleting ${oldReplicas.length} old replica(s)`, {oldReplicas});\n    await sql`DELETE FROM ${sql(replicasTable)} WHERE rank < ${beforeRank}`;\n  }\n\n  return dropUnclaimedSlots(lc, sql, shard);\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;AAgDA,IAAM,qCAAqC;AAK3C,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,0BAEU;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,WAAW,eAAe;CAI1C,MAAM,QAAQ,MAAM,UAHD,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAE/C,GAAY,kCAAkC;CAC5E,IAAI,UAAU,aAAa;EAGzB,QACG,IAAI,EACJ,OAAM,MACL,GAAG,OAAO,oDAAoD,CAAC,CACjE;EACF,MAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,qCAEjG;CACF;CACA,MAAM,CAAC,QAAQ;CACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;CACtD,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;AAyBA,eAAsB,qBACpB,IACA,KACA,oBACA,OACA,WACA,UACA,eAC0B;CAC1B,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAClD,KAAK,IAAI,QAAQ,OAAQ,QAAQ,OAAO;EACtC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EACvC,IAAI;GACF,OAAO,MAAM,MAAM,KAAK,OAAM,OAAM;IAClC,MAAM,EAAE,yCAAyC,SAAS;IAG1D,IAAI;IACJ,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,OAAO;IACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;IAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;KAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;KAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;MAC7B,WAAW;MACX;KACF;IACF;IAEA,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;KAC/D;KACA;IACF,CAAC;IAED,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,qBAAqB,KAAK,gBAAgB,GAC1C,aACF;IAEA,OAAO;GACT,CAAC;EACH,SAAS,GAAG;GACV,IAAI,SAAS,gBAAgB,GAAG,yBAAyB,GAAG;IAK1D,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD;GACF;GAIA,IAAI,SAAS,gBAAgB,GAAG,+BAA+B,GAAG;IAChE,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC;GACF;GACA,MAAM;EACR;CACF;AACF;;;;;;;;;;;AAYA,eAAsB,wBACpB,IACA,KACA,OACA,YAC8D;CAC9D,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAC/C,MAAM,cAAc,MAAM,GAAG;;YAEnB,IAAI,aAAa,EAAE,gBAAgB,WAAW;;CAExD,IAAI,YAAY,QAAQ;EACtB,GAAG,OAAO,YAAY,YAAY,OAAO,kBAAkB,EAAC,YAAW,CAAC;EACxE,MAAM,GAAG,eAAe,IAAI,aAAa,EAAE,gBAAgB;CAC7D;CAEA,OAAO,mBAAmB,IAAI,KAAK,KAAK;AAC1C;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAG3D,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}
+{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type postgres from 'postgres';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {\n  createReplicationSessionFor,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  createReplica,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: must be false if pgVersion < PG_17. Caller must verify.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nconst CREATE_REPLICATION_SLOT_TIMEOUT_MS = 30_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n  if (raced === 'timed-out') {\n    // Create slot can block indefinitely waiting for old transactions. End\n    // this connection in the background and fail fast so the process restarts.\n    void session\n      .end()\n      .catch(e =>\n        lc.warn?.(`Error closing timed out replication slot session`, e),\n      );\n    throw new Error(\n      `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}. ` +\n        `Crashing to force a clean restart.`,\n    );\n  }\n  const [slot] = raced;\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n): Promise<ReplicationSlotResult<T>> {\n  // Note: The replicationSession is closed by keepSlotActiveUntilTakenOver,\n  // and only closed in this function if an error occurrs.\n  const replicationSession = createReplicationSessionFor(sql, sessionName);\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n\n  for (let first = true; ; first = false) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    let slotName: string | undefined;\n    try {\n      return await runTx(sql, async tx => {\n        await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n        // Pick an available slotName from the slotPoolPrefix pool.\n        const names = await tx<{name: string}[]> /*sql*/ `\n          SELECT slot_name as name FROM pg_replication_slots\n            WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n        `.values();\n        const inUse = new Set(names.flat());\n        for (let next = 0; ; next++) {\n          const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n          if (!inUse.has(candidateName)) {\n            slotName = candidateName;\n            break;\n          }\n        }\n\n        const slot = await createReplicationSlot(lc, replicationSession, {\n          slotName,\n          failover,\n        });\n        const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n        const initialSession = await keepSlotActiveUntilTakenOver(\n          lc,\n          replicationSession,\n          slot.slot_name,\n          metadataPublicationName(shard.appID, shard.shardNum),\n          toBigInt(slot.consistent_point),\n        );\n\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          toStateVersionString(slot.consistent_point),\n          backupOptions,\n        );\n\n        return {slot, capturedSnapshot, initialSession};\n      });\n    } catch (e) {\n      if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      // Otherwise, clean up any created slot if something went wrong.\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await replicationSession.end();\n        await sql`SELECT pg_drop_replication_slot(${slotName})`;\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Deletes \"old\" replicas (i.e. those with a lower rank than the current)\n * and attempts to drop replication slots that are not associated with any\n * replica.\n *\n * If a slot could not be dropped because there is still an active subscriber,\n * it will be reflected in the `draining` count that is returned. When there\n * are draining slots, the method should be retried until all orphaned slots\n * have been dropped.\n */\nexport async function dropOldReplicasAndSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  beforeRank: bigint,\n): Promise<{dropped: number; active: number; draining: number}> {\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n  const oldReplicas = await sql`\n    SELECT id, rank::float8, slot, version, \"initialSyncContext\", \"subscriberContext\"\n     FROM ${sql(replicasTable)} WHERE rank < ${beforeRank};\n  `;\n  if (oldReplicas.length) {\n    lc.info?.(`Deleting ${oldReplicas.length} old replica(s)`, {oldReplicas});\n    await sql`DELETE FROM ${sql(replicasTable)} WHERE rank < ${beforeRank}`;\n  }\n\n  return dropUnclaimedSlots(lc, sql, shard);\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;AA4DA,IAAM,qCAAqC;AAK3C,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,0BAEU;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,WAAW,eAAe;CAI1C,MAAM,QAAQ,MAAM,UAHD,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAE/C,GAAY,kCAAkC;CAC5E,IAAI,UAAU,aAAa;EAGzB,QACG,IAAI,EACJ,OAAM,MACL,GAAG,OAAO,oDAAoD,CAAC,CACjE;EACF,MAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,qCAEjG;CACF;CACA,MAAM,CAAC,QAAQ;CACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;CACtD,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0CA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,WACA,UACA,eACA,iBACmC;CAGnC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAElD,KAAK,IAAI,QAAQ,OAAQ,QAAQ,OAAO;EACtC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAEvC,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,KAAK,OAAM,OAAM;IAClC,MAAM,EAAE,yCAAyC,SAAS;IAG1D,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,OAAO;IACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;IAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;KAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;KAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;MAC7B,WAAW;MACX;KACF;IACF;IAEA,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;KAC/D;KACA;IACF,CAAC;IACD,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;IACjE,MAAM,iBAAiB,MAAM,6BAC3B,IACA,oBACA,KAAK,WACL,wBAAwB,MAAM,OAAO,MAAM,QAAQ,GACnD,SAAS,KAAK,gBAAgB,CAChC;IAEA,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,qBAAqB,KAAK,gBAAgB,GAC1C,aACF;IAEA,OAAO;KAAC;KAAM;KAAkB;IAAc;GAChD,CAAC;EACH,SAAS,GAAG;GACV,IAAI,SAAS,gBAAgB,GAAG,yBAAyB,GAAG;IAK1D,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD;GACF;GAIA,IAAI,SAAS,gBAAgB,GAAG,+BAA+B,GAAG;IAChE,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC;GACF;GAEA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI;IAC7B,MAAM,GAAG,mCAAmC,SAAS;GACvD;GACA,MAAM;EACR;CACF;AACF;;;;;;;;;;;AAYA,eAAsB,wBACpB,IACA,KACA,OACA,YAC8D;CAC9D,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAC/C,MAAM,cAAc,MAAM,GAAG;;YAEnB,IAAI,aAAa,EAAE,gBAAgB,WAAW;;CAExD,IAAI,YAAY,QAAQ;EACtB,GAAG,OAAO,YAAY,YAAY,OAAO,kBAAkB,EAAC,YAAW,CAAC;EACxE,MAAM,GAAG,eAAe,IAAI,aAAa,EAAE,gBAAgB;CAC7D;CAEA,OAAO,mBAAmB,IAAI,KAAK,KAAK;AAC1C;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAG3D,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}