@rocicorp/zero 0.26.1 → 0.26.2-canary.4

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js

@@ -1,163 +1,121 @@
-import { PG_ADMIN_SHUTDOWN, PG_OBJECT_IN_USE, PG_OBJECT_NOT_IN_PREREQUISITE_STATE } from "@drdgvhbh/postgres-error-codes";
-import { defu } from "defu";
-import postgres from "postgres";
 import { sleep } from "../../../../../../shared/src/sleep.js";
-import { getTypeParsers } from "../../../../db/pg-type-parser.js";
 import "../../../../types/pg.js";
+import { fromBigInt } from "../lsn.js";
 import { id, lit } from "../../../../types/sql.js";
-import { pipe } from "../../../../types/streams.js";
 import { Subscription } from "../../../../types/subscription.js";
+import { pipe } from "../../../../types/streams.js";
 import { AutoResetSignal } from "../../../change-streamer/schema/tables.js";
-import { fromBigInt } from "../lsn.js";
+import { getTypeParsers } from "../../../../db/pg-type-parser.js";
 import { PgoutputParser } from "./pgoutput-parser.js";
-const DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;
+import postgres from "postgres";
+import { defu } from "defu";
+import { PG_ADMIN_SHUTDOWN, PG_OBJECT_IN_USE, PG_OBJECT_NOT_IN_PREREQUISITE_STATE } from "@drdgvhbh/postgres-error-codes";
+//#region ../zero-cache/src/services/change-source/pg/logical-replication/stream.ts
+var DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;
 async function subscribe(lc, db, slot, publications, lsn, retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE, applicationName = "zero-replicator") {
-  const session = postgres(
-    defu(
-      {
-        max: 1,
-        ["fetch_types"]: false,
-        // Necessary for the streaming protocol
-        ["idle_timeout"]: null,
-        ["max_lifetime"]: null,
-        connection: {
-          ["application_name"]: applicationName,
-          replication: "database"
-          // https://www.postgresql.org/docs/current/protocol-replication.html
-        }
-      },
-      // ParsedOptions are technically compatible with Options, but happen
-      // to not be typed that way. The postgres.js author does an equivalent
-      // merge of ParsedOptions and Options here:
-      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13
-      db.options
-    )
-  );
-  const [{ walSenderTimeoutMs }] = await session`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 
+	const session = postgres(defu({
+		max: 1,
+		["fetch_types"]: false,
+		["idle_timeout"]: null,
+		["max_lifetime"]: null,
+		connection: {
+			["application_name"]: applicationName,
+			replication: "database"
+		}
+	}, db.options));
+	const [{ walSenderTimeoutMs }] = await session`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 
         AS "walSenderTimeoutMs" FROM pg_settings
         WHERE name = 'wal_sender_timeout'`.simple();
-  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);
-  lc.info?.(
-    `wal_sender_timeout: ${walSenderTimeoutMs}ms. Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms`
-  );
-  const [readable, writable] = await startReplicationStream(
-    lc,
-    session,
-    slot,
-    publications,
-    lsn,
-    retriesIfReplicationSlotActive + 1
-  );
-  let lastAckTime = Date.now();
-  function sendAck(lsn2) {
-    writable.write(makeAck(lsn2));
-    lastAckTime = Date.now();
-  }
-  const livenessTimer = setInterval(() => {
-    const now = Date.now();
-    if (now - lastAckTime > manualKeepaliveTimeout) {
-      sendAck(0n);
-      lc.debug?.(`sent manual keepalive`);
-    }
-  }, manualKeepaliveTimeout / 5);
-  let destroyed = false;
-  const typeParsers = await getTypeParsers(db, { returnJsonAsString: true });
-  const parser = new PgoutputParser(typeParsers);
-  const messages = Subscription.create({
-    cleanup: () => {
-      destroyed = true;
-      readable.destroyed || readable.destroy();
-      clearInterval(livenessTimer);
-      return session.end();
-    }
-  });
-  readable.once(
-    "close",
-    () => (
-      // Only log a warning if the stream was not manually closed.
-      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`)
-    )
-  );
-  readable.once(
-    "error",
-    (e) => (
-      // Don't log the shutdown signal. This is the expected way for upstream
-      // to close the connection (and will be logged downstream).
-      e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN || lc.warn?.(`error from ${db.options.host}`, e)
-    )
-  );
-  pipe({
-    source: readable,
-    sink: messages,
-    parse: (buffer) => parseStreamMessage(lc, buffer, parser),
-    // Allow a small buffer of messages to be queued in the subscription so
-    // that the change-source loop can check the queue to determine if more
-    // messages are immediately available.
-    bufferMessages: 5
-  });
-  return {
-    messages,
-    acks: { push: sendAck }
-  };
+	const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * .75);
+	lc.info?.(`wal_sender_timeout: ${walSenderTimeoutMs}ms. Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms`);
+	const [readable, writable] = await startReplicationStream(lc, session, slot, publications, lsn, retriesIfReplicationSlotActive + 1);
+	let lastAckTime = Date.now();
+	function sendAck(lsn) {
+		writable.write(makeAck(lsn));
+		lastAckTime = Date.now();
+	}
+	const livenessTimer = setInterval(() => {
+		if (Date.now() - lastAckTime > manualKeepaliveTimeout) {
+			sendAck(0n);
+			lc.debug?.(`sent manual keepalive`);
+		}
+	}, manualKeepaliveTimeout / 5);
+	let destroyed = false;
+	const parser = new PgoutputParser(await getTypeParsers(db, { returnJsonAsString: true }));
+	const messages = Subscription.create({ cleanup: () => {
+		destroyed = true;
+		readable.destroyed || readable.destroy();
+		clearInterval(livenessTimer);
+		return session.end();
+	} });
+	readable.once("close", () => destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`));
+	readable.once("error", (e) => e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN || lc.warn?.(`error from ${db.options.host}`, e));
+	pipe({
+		source: readable,
+		sink: messages,
+		parse: (buffer) => parseStreamMessage(lc, buffer, parser),
+		bufferMessages: 5
+	});
+	return {
+		messages,
+		acks: { push: sendAck }
+	};
 }
+/**
+* Formats publication names for the START_REPLICATION command.
+* The replication protocol expects format: publication_names 'pub1,pub2'
+* Each name is escaped to handle any quotes that may have passed validation.
+*/
 function formatPublicationNames(publications) {
-  return publications.map((p) => lit(p).slice(1, -1)).join(",");
+	return publications.map((p) => lit(p).slice(1, -1)).join(",");
 }
 async function startReplicationStream(lc, session, slot, publications, lsn, maxAttempts) {
-  for (let i = 0; i < maxAttempts; i++) {
-    try {
-      const stream = session.unsafe(
-        `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (
+	for (let i = 0; i < maxAttempts; i++) try {
+		const stream = session.unsafe(`START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (
         proto_version '1',
         publication_names '${formatPublicationNames(publications)}',
         messages 'true'
-      )`
-      ).execute();
-      return await Promise.all([stream.readable(), stream.writable()]);
-    } catch (e) {
-      if (e instanceof postgres.PostgresError) {
-        if (e.code === PG_OBJECT_IN_USE) {
-          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);
-          await sleep(10);
-          continue;
-        }
-        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {
-          lc.error?.(`error starting replication stream`, e);
-          throw new AutoResetSignal(`unable to start replication stream`, {
-            cause: e
-          });
-        }
-      }
-      throw e;
-    }
-  }
-  throw new Error(
-    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`
-  );
+      )`).execute();
+		return await Promise.all([stream.readable(), stream.writable()]);
+	} catch (e) {
+		if (e instanceof postgres.PostgresError) {
+			if (e.code === PG_OBJECT_IN_USE) {
+				lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);
+				await sleep(10);
+				continue;
+			}
+			if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {
+				lc.error?.(`error starting replication stream`, e);
+				throw new AutoResetSignal(`unable to start replication stream`, { cause: e });
+			}
+		}
+		throw e;
+	}
+	throw new Error(`exceeded max attempts (${maxAttempts}) to start the Postgres stream`);
 }
 function parseStreamMessage(lc, buffer, parser) {
-  if (buffer[0] !== 119 && buffer[0] !== 107) {
-    lc.warn?.("Unknown message", buffer[0]);
-    return null;
-  }
-  const lsn = buffer.readBigUInt64BE(1);
-  if (buffer[0] === 119) {
-    return [lsn, parser.parse(buffer.subarray(25))];
-  }
-  const shouldRespond = buffer.readInt8(17) !== 0;
-  return [lsn, { tag: "keepalive", shouldRespond }];
+	if (buffer[0] !== 119 && buffer[0] !== 107) {
+		lc.warn?.("Unknown message", buffer[0]);
+		return null;
+	}
+	const lsn = buffer.readBigUInt64BE(1);
+	if (buffer[0] === 119) return [lsn, parser.parse(buffer.subarray(25))];
+	return [lsn, {
+		tag: "keepalive",
+		shouldRespond: buffer.readInt8(17) !== 0
+	}];
 }
 function makeAck(lsn) {
-  const microNow = BigInt(Date.now() - Date.UTC(2e3, 0, 1)) * BigInt(1e3);
-  const x = Buffer.alloc(34);
-  x[0] = "r".charCodeAt(0);
-  x.writeBigInt64BE(lsn, 1);
-  x.writeBigInt64BE(lsn, 9);
-  x.writeBigInt64BE(lsn, 17);
-  x.writeBigInt64BE(microNow, 25);
-  return x;
+	const microNow = BigInt(Date.now() - Date.UTC(2e3, 0, 1)) * BigInt(1e3);
+	const x = Buffer.alloc(34);
+	x[0] = "r".charCodeAt(0);
+	x.writeBigInt64BE(lsn, 1);
+	x.writeBigInt64BE(lsn, 9);
+	x.writeBigInt64BE(lsn, 17);
+	x.writeBigInt64BE(microNow, 25);
+	return x;
 }
-export {
-  subscribe
-};
-//# sourceMappingURL=stream.js.map
+//#endregion
+export { subscribe };
+
+//# sourceMappingURL=stream.js.map

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream.js","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  lc.info?.(\n    `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n      `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  const livenessTimer = setInterval(() => {\n    const now = Date.now();\n    if (now - lastAckTime > manualKeepaliveTimeout) {\n      sendAck(0n);\n      lc.debug?.(`sent manual keepalive`);\n    }\n  }, manualKeepaliveTimeout / 5);\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => parseStreamMessage(lc, buffer, parser),\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"names":["lsn"],"mappings":";;;;;;;;;;;;AAmBA,MAAM,6CAA6C;AAcnD,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAIjB;AACD,QAAM,UAAU;AAAA,IACd;AAAA,MACE;AAAA,QACE,KAAK;AAAA,QACL,CAAC,aAAa,GAAG;AAAA;AAAA,QACjB,CAAC,cAAc,GAAG;AAAA,QAClB,CAAC,cAAc,GAAG;AAAA,QAClB,YAAY;AAAA,UACV,CAAC,kBAAkB,GAAG;AAAA,UACtB,aAAa;AAAA;AAAA,QAAA;AAAA,MACf;AAAA;AAAA;AAAA;AAAA;AAAA,MAMF,GAAG;AAAA,IAAA;AAAA,EACL;AASF,QAAM,CAAC,EAAC,oBAAmB,IAAI,MAAM;AAAA;AAAA,2CAII,OAAA;AACzC,QAAM,yBAAyB,KAAK,MAAM,qBAAqB,IAAI;AACnE,KAAG;AAAA,IACD,uBAAuB,kBAAkB,iDACM,sBAAsB;AAAA,EAAA;AAGvE,QAAM,CAAC,UAAU,QAAQ,IAAI,MAAM;AAAA,IACjC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,iCAAiC;AAAA,EAAA;AAGnC,MAAI,cAAc,KAAK,IAAA;AACvB,WAAS,QAAQA,MAAa;AAC5B,aAAS,MAAM,QAAQA,IAAG,CAAC;AAC3B,kBAAc,KAAK,IAAA;AAAA,EACrB;AAEA,QAAM,gBAAgB,YAAY,MAAM;AACtC,UAAM,MAAM,KAAK,IAAA;AACjB,QAAI,MAAM,cAAc,wBAAwB;AAC9C,cAAQ,EAAE;AACV,SAAG,QAAQ,uBAAuB;AAAA,IACpC;AAAA,EACF,GAAG,yBAAyB,CAAC;AAE7B,MAAI,YAAY;AAChB,QAAM,cAAc,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK;AACvE,QAAM,SAAS,IAAI,eAAe,WAAW;AAC7C,QAAM,WAAW,aAAa,OAAsB;AAAA,IAClD,SAAS,MAAM;AACb,kBAAY;AACZ,eAAS,aAAa,SAAS,QAAA;AAC/B,oBAAc,aAAa;AAC3B,aAAO,QAAQ,IAAA;AAAA,IACjB;AAAA,EAAA,CACD;AAED,WAAS;AAAA,IACP;AAAA,IACA;AAAA;AAAA,MAEE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,IAAI,EAAE;AAAA;AAAA,EAAA;AAE5E,WAAS;AAAA,IACP;AAAA,IACA,CAAA;AAAA;AAAA;AAAA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,IAAI,IAAI,CAAC;AAAA;AAAA,EAAA;AAGhD,OAAK;AAAA,IACH,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,OAAO,CAAA,WAAU,mBAAmB,IAAI,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA,IAItD,gBAAgB;AAAA,EAAA,CACjB;AAED,SAAO;AAAA,IACL;AAAA,IACA,MAAM,EAAC,MAAM,QAAA;AAAA,EAAO;AAExB;AAOA,SAAS,uBAAuB,cAAgC;AAG9D,SAAO,aAAa,IAAI,CAAA,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;AACA,WAAS,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,QAAI;AACF,YAAM,SAAS,QACZ;AAAA,QACC,0BAA0B,GAAG,IAAI,CAAC,YAAY,WAAW,GAAG,CAAC;AAAA;AAAA,6BAE1C,uBAAuB,YAAY,CAAC;AAAA;AAAA;AAAA,MAAA,EAIxD,QAAA;AACH,aAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,YAAY,OAAO,SAAA,CAAU,CAAC;AAAA,IACjE,SAAS,GAAG;AACV,UAAI,aAAa,SAAS,eAAe;AAEvC,YAAI,EAAE,SAAS,kBAAkB;AAI/B,aAAG,OAAO,WAAW,IAAI,CAAC,KAAK,OAAO,CAAC,CAAC,IAAI,CAAC;AAC7C,gBAAM,MAAM,EAAE;AACd;AAAA,QACF;AAIA,YAAI,EAAE,SAAS,qCAAqC;AAClD,aAAG,QAAQ,qCAAqC,CAAC;AACjD,gBAAM,IAAI,gBAAgB,sCAAsC;AAAA,YAC9D,OAAO;AAAA,UAAA,CACR;AAAA,QACH;AAAA,MACF;AACA,YAAM;AAAA,IACR;AAAA,EACF;AACA,QAAM,IAAI;AAAA,IACR,0BAA0B,WAAW;AAAA,EAAA;AAEzC;AAEA,SAAS,mBACP,IACA,QACA,QACsB;AAEtB,MAAI,OAAO,CAAC,MAAM,OAAQ,OAAO,CAAC,MAAM,KAAM;AAC5C,OAAG,OAAO,mBAAmB,OAAO,CAAC,CAAC;AACtC,WAAO;AAAA,EACT;AACA,QAAM,MAAM,OAAO,gBAAgB,CAAC;AACpC,MAAI,OAAO,CAAC,MAAM,KAAM;AAGtB,WAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;AAAA,EAChD;AAGA,QAAM,gBAAgB,OAAO,SAAS,EAAE,MAAM;AAC9C,SAAO,CAAC,KAAK,EAAC,KAAK,aAAa,eAAc;AAChD;AAGA,SAAS,QAAQ,KAAqB;AACpC,QAAM,WAAW,OAAO,KAAK,IAAA,IAAQ,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;AAExE,QAAM,IAAI,OAAO,MAAM,EAAE;AACzB,IAAE,CAAC,IAAI,IAAI,WAAW,CAAC;AACvB,IAAE,gBAAgB,KAAK,CAAC;AACxB,IAAE,gBAAgB,KAAK,CAAC;AACxB,IAAE,gBAAgB,KAAK,EAAE;AACzB,IAAE,gBAAgB,UAAU,EAAE;AAC9B,SAAO;AACT;"}
+{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  lc.info?.(\n    `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n      `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  const livenessTimer = setInterval(() => {\n    const now = Date.now();\n    if (now - lastAckTime > manualKeepaliveTimeout) {\n      sendAck(0n);\n      lc.debug?.(`sent manual keepalive`);\n    }\n  }, manualKeepaliveTimeout / 5);\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => parseStreamMessage(lc, buffer, parser),\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;AAmBA,IAAM,6CAA6C;AAcnD,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAIjB;CACD,MAAM,UAAU,SACd,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;GACd;EACF,EAKD,GAAG,QACJ,CACF;CAQD,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,QAAQ;CACjD,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,IAAK;AACpE,IAAG,OACD,uBAAuB,mBAAmB,gDACK,uBAAuB,IACvE;CAED,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,EAClC;CAED,IAAI,cAAc,KAAK,KAAK;CAC5B,SAAS,QAAQ,KAAa;AAC5B,WAAS,MAAM,QAAQ,IAAI,CAAC;AAC5B,gBAAc,KAAK,KAAK;;CAG1B,MAAM,gBAAgB,kBAAkB;AAEtC,MADY,KAAK,KAAK,GACZ,cAAc,wBAAwB;AAC9C,WAAQ,GAAG;AACX,MAAG,QAAQ,wBAAwB;;IAEpC,yBAAyB,EAAE;CAE9B,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eADC,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK,CAAC,CAC1B;CAC9C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;AACb,cAAY;AACZ,WAAS,aAAa,SAAS,SAAS;AACxC,gBAAc,cAAc;AAC5B,SAAO,QAAQ,KAAK;IAEvB,CAAC;AAEF,UAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,OAAO,CAC5E;AACD,UAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,EAAE,CAChD;AAED,MAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU,mBAAmB,IAAI,QAAQ,OAAO;EAIvD,gBAAgB;EACjB,CAAC;AAEF,QAAO;EACL;EACA,MAAM,EAAC,MAAM,SAAQ;EACtB;;;;;;;AAQH,SAAS,uBAAuB,cAAgC;AAG9D,QAAO,aAAa,KAAI,MAAK,IAAI,EAAE,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,KAAK,IAAI;;AAG7D,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;AACA,MAAK,IAAI,IAAI,GAAG,IAAI,aAAa,IAC/B,KAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,KAAK,CAAC,WAAW,WAAW,IAAI,CAAC;;6BAE3C,uBAAuB,aAAa,CAAC;;SAGzD,CACA,SAAS;AACZ,SAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,UAAU,EAAE,OAAO,UAAU,CAAC,CAAC;UACzD,GAAG;AACV,MAAI,aAAa,SAAS,eAAe;AAEvC,OAAI,EAAE,SAAS,kBAAkB;AAI/B,OAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,EAAE,IAAI,EAAE;AAC9C,UAAM,MAAM,GAAG;AACf;;AAKF,OAAI,EAAE,SAAS,qCAAqC;AAClD,OAAG,QAAQ,qCAAqC,EAAE;AAClD,UAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,GACR,CAAC;;;AAGN,QAAM;;AAGV,OAAM,IAAI,MACR,0BAA0B,YAAY,gCACvC;;AAGH,SAAS,mBACP,IACA,QACA,QACsB;AAEtB,KAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;AAC5C,KAAG,OAAO,mBAAmB,OAAO,GAAG;AACvC,SAAO;;CAET,MAAM,MAAM,OAAO,gBAAgB,EAAE;AACrC,KAAI,OAAO,OAAO,IAGhB,QAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,GAAG,CAAC,CAAC;AAKjD,QAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,GAAG,KAAK;EACA,CAAC;;AAIjD,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,KAAK,GAAG,KAAK,IAAI,KAAM,GAAG,EAAE,CAAC,GAAG,OAAO,IAAK;CAEzE,MAAM,IAAI,OAAO,MAAM,GAAG;AAC1B,GAAE,KAAK,IAAI,WAAW,EAAE;AACxB,GAAE,gBAAgB,KAAK,EAAE;AACzB,GAAE,gBAAgB,KAAK,EAAE;AACzB,GAAE,gBAAgB,KAAK,GAAG;AAC1B,GAAE,gBAAgB,UAAU,GAAG;AAC/B,QAAO"}

package/out/zero-cache/src/services/change-source/pg/lsn.js

@@ -1,29 +1,27 @@
 import { assert } from "../../../../../shared/src/asserts.js";
 import { majorVersionToString, stateVersionFromString } from "../../../types/state-version.js";
+//#region ../zero-cache/src/services/change-source/pg/lsn.ts
 function toBigInt(lsn) {
-  const parts = lsn.split("/");
-  assert(parts.length === 2, `Malformed LSN: "${lsn}"`);
-  const high = BigInt(`0x${parts[0]}`);
-  const low = BigInt(`0x${parts[1]}`);
-  const val = (high << 32n) + low;
-  return val;
+	const parts = lsn.split("/");
+	assert(parts.length === 2, `Malformed LSN: "${lsn}"`);
+	const high = BigInt(`0x${parts[0]}`);
+	const low = BigInt(`0x${parts[1]}`);
+	return (high << 32n) + low;
 }
 function toStateVersionString(lsn) {
-  return majorVersionToString(toBigInt(lsn));
+	return majorVersionToString(toBigInt(lsn));
 }
+/** The LSN is tracked by the `major` component of the StateVersion. */
 function fromStateVersionString(ver) {
-  const { major } = stateVersionFromString(ver);
-  return fromBigInt(major);
+	const { major } = stateVersionFromString(ver);
+	return fromBigInt(major);
 }
 function fromBigInt(val) {
-  const high = val >> 32n;
-  const low = val & 0xffffffffn;
-  return `${high.toString(16).toUpperCase()}/${low.toString(16).toUpperCase()}`;
+	const high = val >> 32n;
+	const low = val & 4294967295n;
+	return `${high.toString(16).toUpperCase()}/${low.toString(16).toUpperCase()}`;
 }
-export {
-  fromBigInt,
-  fromStateVersionString,
-  toBigInt,
-  toStateVersionString
-};
-//# sourceMappingURL=lsn.js.map
+//#endregion
+export { fromBigInt, fromStateVersionString, toStateVersionString };
+
+//# sourceMappingURL=lsn.js.map

package/out/zero-cache/src/services/change-source/pg/lsn.js.map

@@ -1 +1 @@
-{"version":3,"file":"lsn.js","sources":["../../../../../../../zero-cache/src/services/change-source/pg/lsn.ts"],"sourcesContent":["import {assert} from '../../../../../shared/src/asserts.ts';\nimport {\n  majorVersionToString,\n  stateVersionFromString,\n} from '../../../types/state-version.ts';\nimport type {Change} from '../protocol/current/data.ts';\n\n/**\n * Parsing and conversion utilities for the pg_lsn Type, which represents\n * the \"Log Sequence Number\" used as a monotonic progress marker for logical\n * replication from a PostgresSQL database.\n *\n * The LSN is a 64-bit integer represented in logical replication as two\n * hexadecimal numbers (up to 8 digits each) separated by a slash. This is\n * converted to a LexiVersion and used as DB-agnostic version in change log,\n * invalidation index, and row version in the tables of the sync replica.\n */\nexport type LSN = string;\n\nexport type RecordType = Change['tag'];\n\nexport function toBigInt(lsn: LSN): bigint {\n  const parts = lsn.split('/');\n  assert(parts.length === 2, `Malformed LSN: \"${lsn}\"`);\n  const high = BigInt(`0x${parts[0]}`);\n  const low = BigInt(`0x${parts[1]}`);\n  const val = (high << 32n) + low;\n  return val;\n}\n\nexport function toStateVersionString(lsn: LSN): string {\n  return majorVersionToString(toBigInt(lsn));\n}\n\n/** The LSN is tracked by the `major` component of the StateVersion. */\nexport function fromStateVersionString(ver: string): LSN {\n  const {major} = stateVersionFromString(ver);\n  return fromBigInt(major);\n}\n\nexport function fromBigInt(val: bigint): LSN {\n  const high = val >> 32n;\n  const low = val & 0xffffffffn;\n  return `${high.toString(16).toUpperCase()}/${low.toString(16).toUpperCase()}`;\n}\n"],"names":[],"mappings":";;AAqBO,SAAS,SAAS,KAAkB;AACzC,QAAM,QAAQ,IAAI,MAAM,GAAG;AAC3B,SAAO,MAAM,WAAW,GAAG,mBAAmB,GAAG,GAAG;AACpD,QAAM,OAAO,OAAO,KAAK,MAAM,CAAC,CAAC,EAAE;AACnC,QAAM,MAAM,OAAO,KAAK,MAAM,CAAC,CAAC,EAAE;AAClC,QAAM,OAAO,QAAQ,OAAO;AAC5B,SAAO;AACT;AAEO,SAAS,qBAAqB,KAAkB;AACrD,SAAO,qBAAqB,SAAS,GAAG,CAAC;AAC3C;AAGO,SAAS,uBAAuB,KAAkB;AACvD,QAAM,EAAC,MAAA,IAAS,uBAAuB,GAAG;AAC1C,SAAO,WAAW,KAAK;AACzB;AAEO,SAAS,WAAW,KAAkB;AAC3C,QAAM,OAAO,OAAO;AACpB,QAAM,MAAM,MAAM;AAClB,SAAO,GAAG,KAAK,SAAS,EAAE,EAAE,YAAA,CAAa,IAAI,IAAI,SAAS,EAAE,EAAE,aAAa;AAC7E;"}
+{"version":3,"file":"lsn.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/lsn.ts"],"sourcesContent":["import {assert} from '../../../../../shared/src/asserts.ts';\nimport {\n  majorVersionToString,\n  stateVersionFromString,\n} from '../../../types/state-version.ts';\nimport type {Change} from '../protocol/current/data.ts';\n\n/**\n * Parsing and conversion utilities for the pg_lsn Type, which represents\n * the \"Log Sequence Number\" used as a monotonic progress marker for logical\n * replication from a PostgresSQL database.\n *\n * The LSN is a 64-bit integer represented in logical replication as two\n * hexadecimal numbers (up to 8 digits each) separated by a slash. This is\n * converted to a LexiVersion and used as DB-agnostic version in change log,\n * invalidation index, and row version in the tables of the sync replica.\n */\nexport type LSN = string;\n\nexport type RecordType = Change['tag'];\n\nexport function toBigInt(lsn: LSN): bigint {\n  const parts = lsn.split('/');\n  assert(parts.length === 2, `Malformed LSN: \"${lsn}\"`);\n  const high = BigInt(`0x${parts[0]}`);\n  const low = BigInt(`0x${parts[1]}`);\n  const val = (high << 32n) + low;\n  return val;\n}\n\nexport function toStateVersionString(lsn: LSN): string {\n  return majorVersionToString(toBigInt(lsn));\n}\n\n/** The LSN is tracked by the `major` component of the StateVersion. */\nexport function fromStateVersionString(ver: string): LSN {\n  const {major} = stateVersionFromString(ver);\n  return fromBigInt(major);\n}\n\nexport function fromBigInt(val: bigint): LSN {\n  const high = val >> 32n;\n  const low = val & 0xffffffffn;\n  return `${high.toString(16).toUpperCase()}/${low.toString(16).toUpperCase()}`;\n}\n"],"mappings":";;;AAqBA,SAAgB,SAAS,KAAkB;CACzC,MAAM,QAAQ,IAAI,MAAM,IAAI;AAC5B,QAAO,MAAM,WAAW,GAAG,mBAAmB,IAAI,GAAG;CACrD,MAAM,OAAO,OAAO,KAAK,MAAM,KAAK;CACpC,MAAM,MAAM,OAAO,KAAK,MAAM,KAAK;AAEnC,SADa,QAAQ,OAAO;;AAI9B,SAAgB,qBAAqB,KAAkB;AACrD,QAAO,qBAAqB,SAAS,IAAI,CAAC;;;AAI5C,SAAgB,uBAAuB,KAAkB;CACvD,MAAM,EAAC,UAAS,uBAAuB,IAAI;AAC3C,QAAO,WAAW,MAAM;;AAG1B,SAAgB,WAAW,KAAkB;CAC3C,MAAM,OAAO,OAAO;CACpB,MAAM,MAAM,MAAM;AAClB,QAAO,GAAG,KAAK,SAAS,GAAG,CAAC,aAAa,CAAC,GAAG,IAAI,SAAS,GAAG,CAAC,aAAa"}

package/out/zero-cache/src/services/change-source/pg/schema/ddl.js

@@ -1,38 +1,55 @@
-import { literal as literal$1 } from "pg-format";
 import { assert } from "../../../../../../shared/src/asserts.js";
-import "../../../../../../shared/src/valita.js";
+import { valita_exports } from "../../../../../../shared/src/valita.js";
 import { upstreamSchema } from "../../../../types/shards.js";
+import { indexDefinitionsQuery, publishedSchema, publishedTableQuery } from "./published.js";
 import { id } from "../../../../types/sql.js";
-import { publishedSchema, publishedTableQuery, indexDefinitionsQuery } from "./published.js";
-import { object, string, literal, union } from "@badrap/valita";
-const PROTOCOL_VERSION = 1;
-const triggerEvent = object({
-  context: object({ query: string() }).rest(string())
+import { literal } from "pg-format";
+var ddlEventSchema = valita_exports.object({ context: valita_exports.object({ query: valita_exports.string() }).rest(valita_exports.string()) }).extend({
+	version: valita_exports.literal(1),
+	schema: publishedSchema
 });
-const ddlEventSchema = triggerEvent.extend({
-  version: literal(PROTOCOL_VERSION),
-  schema: publishedSchema
+var ddlStartEventSchema = ddlEventSchema.extend({ type: valita_exports.literal("ddlStart") });
+/**
+* The {@link DdlUpdateEvent} contains an updated schema resulting from
+* a particular ddl event. The event type provides information
+* (i.e. constraints) on the difference from the schema of the preceding
+* {@link DdlStartEvent}.
+*
+* Note that in almost all cases (the exception being `CREATE` events),
+* it is possible that there is no relevant difference between the
+* ddl-start schema and the ddl-update schema, as many aspects of the
+* schema (e.g. column constraints) are not relevant to downstream
+* replication.
+*/
+var ddlUpdateEventSchema = ddlEventSchema.extend({
+	type: valita_exports.literal("ddlUpdate"),
+	event: valita_exports.object({ tag: valita_exports.string() })
 });
-const ddlStartEventSchema = ddlEventSchema.extend({
-  type: literal("ddlStart")
-});
-const ddlUpdateEventSchema = ddlEventSchema.extend({
-  type: literal("ddlUpdate"),
-  event: object({ tag: string() })
-});
-const replicationEventSchema = union(
-  ddlStartEventSchema,
-  ddlUpdateEventSchema
-);
+var replicationEventSchema = valita_exports.union(ddlStartEventSchema, ddlUpdateEventSchema);
 function append(shardNum) {
-  return (name) => id(name + "_" + String(shardNum));
+	return (name) => id(name + "_" + String(shardNum));
 }
+/**
+* Event trigger functions contain the core logic that are invoked by triggers.
+*
+* Note that although many of these functions can theoretically be parameterized and
+* shared across shards, it is advantageous to keep the functions in each shard
+* isolated from each other in order to avoid the complexity of shared-function
+* versioning.
+*
+* In a sense, shards (and their triggers and functions) should be thought of as
+* execution environments that can be updated at different schedules. If per-shard
+* triggers called into shared functions, we would have to consider versioning the
+* functions when changing their behavior, backwards compatibility, removal of
+* unused versions, etc. (not unlike versioning of npm packages).
+*
+* Instead, we opt for the simplicity and isolation of having each shard
+* completely own (and maintain) the entirety of its trigger/function stack.
+*/
 function createEventFunctionStatements(shard) {
-  const { appID, shardNum, publications } = shard;
-  const schema = id(upstreamSchema(shard));
-  return (
-    /*sql*/
-    `
+	const { appID, shardNum, publications } = shard;
+	const schema = id(upstreamSchema(shard));
+	return `
 CREATE SCHEMA IF NOT EXISTS ${schema};
 
 CREATE OR REPLACE FUNCTION ${schema}.get_trigger_context()
@@ -49,7 +66,7 @@ $$ LANGUAGE plpgsql;
 CREATE OR REPLACE FUNCTION ${schema}.notice_ignore(tag TEXT, target record)
 RETURNS void AS $$
 BEGIN
-  RAISE NOTICE 'zero(%) ignoring % %', ${literal$1(shardNum)}, tag, row_to_json(target);
+  RAISE NOTICE 'zero(%) ignoring % %', ${literal(shardNum)}, tag, row_to_json(target);
 END
 $$ LANGUAGE plpgsql;
 
@@ -80,14 +97,12 @@ BEGIN
 
   SELECT json_build_object(
     'type', 'ddlStart',
-    'version', ${PROTOCOL_VERSION},
+    'version', 1,
     'schema', schema_specs::json,
     'context', ${schema}.get_trigger_context()
   ) INTO message;
 
-  PERFORM pg_logical_emit_message(true, ${literal$1(
-      `${appID}/${shardNum}`
-    )}, message);
+  PERFORM pg_logical_emit_message(true, ${literal(`${appID}/${shardNum}`)}, message);
 END
 $$ LANGUAGE plpgsql;
 
@@ -102,7 +117,7 @@ DECLARE
   message TEXT;
   event TEXT;
 BEGIN
-  publications := ARRAY[${literal$1(publications)}];
+  publications := ARRAY[${literal(publications)}];
 
   SELECT objid, object_type, object_identity 
     FROM pg_event_trigger_ddl_commands() 
@@ -184,95 +199,70 @@ BEGIN
 
   SELECT json_build_object(
     'type', 'ddlUpdate',
-    'version', ${PROTOCOL_VERSION},
+    'version', 1,
     'schema', schema_specs::json,
     'event', event::json,
     'context', ${schema}.get_trigger_context()
   ) INTO message;
 
-  PERFORM pg_logical_emit_message(true, ${literal$1(
-      `${appID}/${shardNum}`
-    )}, message);
+  PERFORM pg_logical_emit_message(true, ${literal(`${appID}/${shardNum}`)}, message);
 END
 $$ LANGUAGE plpgsql;
-`
-  );
+`;
 }
-const TAGS = [
-  "CREATE TABLE",
-  "ALTER TABLE",
-  "CREATE INDEX",
-  "DROP TABLE",
-  "DROP INDEX",
-  "ALTER PUBLICATION",
-  "ALTER SCHEMA"
+var TAGS = [
+	"CREATE TABLE",
+	"ALTER TABLE",
+	"CREATE INDEX",
+	"DROP TABLE",
+	"DROP INDEX",
+	"ALTER PUBLICATION",
+	"ALTER SCHEMA"
 ];
 function createEventTriggerStatements(shard) {
-  assert(shard.publications.length, `shard publications must be non-empty`);
-  const { appID, shardNum } = shard;
-  const sharded = append(shardNum);
-  const schema = id(upstreamSchema(shard));
-  const triggers = [
-    dropEventTriggerStatements(shard.appID, shard.shardNum),
-    createEventFunctionStatements(shard)
-  ];
-  triggers.push(
-    /*sql*/
-    `
+	assert(shard.publications.length, `shard publications must be non-empty`);
+	const { appID, shardNum } = shard;
+	const sharded = append(shardNum);
+	const schema = id(upstreamSchema(shard));
+	const triggers = [dropEventTriggerStatements(shard.appID, shard.shardNum), createEventFunctionStatements(shard)];
+	triggers.push(`
 CREATE EVENT TRIGGER ${sharded(`${appID}_ddl_start`)}
   ON ddl_command_start
-  WHEN TAG IN (${literal$1(TAGS)})
+  WHEN TAG IN (${literal(TAGS)})
   EXECUTE PROCEDURE ${schema}.emit_ddl_start();
-`
-  );
-  for (const tag of TAGS) {
-    const tagID = tag.toLowerCase().replace(" ", "_");
-    triggers.push(
-      /*sql*/
-      `
+`);
+	for (const tag of TAGS) {
+		const tagID = tag.toLowerCase().replace(" ", "_");
+		triggers.push(`
 CREATE OR REPLACE FUNCTION ${schema}.emit_${tagID}() 
 RETURNS event_trigger AS $$
 BEGIN
-  PERFORM ${schema}.emit_ddl_end(${literal$1(tag)});
+  PERFORM ${schema}.emit_ddl_end(${literal(tag)});
 END
 $$ LANGUAGE plpgsql;
 
 CREATE EVENT TRIGGER ${sharded(`${appID}_${tagID}`)}
   ON ddl_command_end
-  WHEN TAG IN (${literal$1(tag)})
+  WHEN TAG IN (${literal(tag)})
   EXECUTE PROCEDURE ${schema}.emit_${tagID}();
-`
-    );
-  }
-  return triggers.join("");
+`);
+	}
+	return triggers.join("");
 }
 function dropEventTriggerStatements(appID, shardID) {
-  const stmts = [];
-  stmts.push(
-    /*sql*/
-    `
+	const stmts = [];
+	stmts.push(`
     DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_start_${shardID}`)};
-  `
-  );
-  for (const tag of TAGS) {
-    const tagID = tag.toLowerCase().replace(" ", "_");
-    stmts.push(
-      /*sql*/
-      `
+  `);
+	for (const tag of TAGS) {
+		const tagID = tag.toLowerCase().replace(" ", "_");
+		stmts.push(`
       DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_${tagID}_${shardID}`)};
-    `
-    );
-  }
-  return stmts.join("");
+    `);
+	}
+	return stmts.join("");
 }
-export {
-  PROTOCOL_VERSION,
-  TAGS,
-  createEventTriggerStatements,
-  ddlEventSchema,
-  ddlStartEventSchema,
-  ddlUpdateEventSchema,
-  dropEventTriggerStatements,
-  replicationEventSchema
-};
-//# sourceMappingURL=ddl.js.map
+//#endregion
+export { createEventTriggerStatements, replicationEventSchema };
+
+//# sourceMappingURL=ddl.js.map