@rocicorp/zero 0.26.1-canary.9 → 0.26.2-canary.1

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js

@@ -1,220 +1,180 @@
-import { PG_UNDEFINED_TABLE, PG_UNDEFINED_COLUMN } from "@drdgvhbh/postgres-error-codes";
-import postgres from "postgres";
-import { equals } from "../../../../../shared/src/set-utils.js";
 import { parse } from "../../../../../shared/src/valita.js";
-import { READONLY } from "../../../db/mode-enum.js";
-import { TsvParser } from "../../../db/pg-copy.js";
-import { getTypeParsers } from "../../../db/pg-type-parser.js";
-import { TransactionPool } from "../../../db/transaction-pool.js";
+import { equals } from "../../../../../shared/src/set-utils.js";
 import { pgClient } from "../../../types/pg.js";
-import { SchemaIncompatibilityError } from "../common/backfill-manager.js";
-import { tableMetadataSchema, columnMetadataSchema } from "./backfill-metadata.js";
-import { makeDownloadStatements, createReplicationSlot } from "./initial-sync.js";
 import { toStateVersionString } from "./lsn.js";
+import { READONLY } from "../../../db/mode-enum.js";
 import { getPublicationInfo } from "./schema/published.js";
-const POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;
+import { SchemaIncompatibilityError } from "../common/backfill-manager.js";
+import { TsvParser } from "../../../db/pg-copy.js";
+import { getTypeParsers } from "../../../db/pg-type-parser.js";
+import { TransactionPool, importSnapshot } from "../../../db/transaction-pool.js";
+import { columnMetadataSchema, tableMetadataSchema } from "./backfill-metadata.js";
+import { createReplicationSlot, makeDownloadStatements } from "./initial-sync.js";
+import postgres from "postgres";
+import { PG_UNDEFINED_COLUMN, PG_UNDEFINED_TABLE } from "@drdgvhbh/postgres-error-codes";
+//#region ../zero-cache/src/services/change-source/pg/backfill-stream.ts
+var POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;
+/**
+* Streams a series of `backfill` messages (ending with `backfill-complete`)
+* at a set watermark (i.e. LSN). The data is retrieved via a COPY stream
+* made at a transaction snapshot corresponding to specific LSN, obtained by
+* creating a short-lived replication slot.
+*/
 async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts = {}) {
-  lc = lc.withContext("component", "backfill").withContext("table", bf.table.name);
-  const { flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE } = opts;
-  const db = pgClient(lc, upstreamURI, {
-    connection: { ["application_name"]: "backfill-stream" },
-    ["max_lifetime"]: 120 * 60
-    // set a long (2h) limit for COPY streaming
-  });
-  const tx = new TransactionPool(lc, READONLY).run(db);
-  try {
-    const watermark = await setSnapshot(lc, upstreamURI, tx, slot);
-    const { tableSpec, backfill } = await validateSchema(
-      tx,
-      publications,
-      bf,
-      watermark
-    );
-    const types = await getTypeParsers(db, { returnJsonAsString: true });
-    const { relation, columns } = backfill;
-    const cols = [...relation.rowKey.columns, ...columns];
-    yield* stream(
-      lc,
-      tx,
-      backfill,
-      makeDownloadStatements(tableSpec, cols),
-      cols.map((col) => types.getTypeParser(tableSpec.columns[col].typeOID)),
-      flushThresholdBytes
-    );
-  } catch (e) {
-    if (e instanceof postgres.PostgresError && (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)) {
-      throw new SchemaIncompatibilityError(bf, String(e), { cause: e });
-    }
-    throw e;
-  } finally {
-    tx.setDone();
-    void tx.done().catch(() => {
-    });
-    void db.end().catch((e) => lc.warn?.(`error closing backfill connection`, e));
-  }
+	lc = lc.withContext("component", "backfill").withContext("table", bf.table.name);
+	const { flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE } = opts;
+	const db = pgClient(lc, upstreamURI, {
+		connection: { ["application_name"]: "backfill-stream" },
+		["max_lifetime"]: 7200
+	});
+	let tx;
+	let watermark;
+	try {
+		({tx, watermark} = await createSnapshotTransaction(lc, upstreamURI, db, slot));
+		const { tableSpec, backfill } = await validateSchema(tx, publications, bf, watermark);
+		const types = await getTypeParsers(db, { returnJsonAsString: true });
+		const { relation, columns } = backfill;
+		const cols = [...relation.rowKey.columns, ...columns];
+		yield* stream(lc, tx, backfill, makeDownloadStatements(tableSpec, cols), cols.map((col) => types.getTypeParser(tableSpec.columns[col].typeOID)), flushThresholdBytes);
+	} catch (e) {
+		if (e instanceof postgres.PostgresError && (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)) throw new SchemaIncompatibilityError(bf, String(e), { cause: e });
+		throw e;
+	} finally {
+		tx?.setDone();
+		db.end().catch((e) => lc.warn?.(`error closing backfill connection`, e));
+	}
 }
 async function* stream(lc, tx, backfill, { select, getTotalRows, getTotalBytes }, colParsers, flushThresholdBytes) {
-  const start = performance.now();
-  const [rows, bytes] = await tx.processReadTask(
-    (sql) => Promise.all([
-      sql.unsafe(getTotalRows),
-      sql.unsafe(getTotalBytes)
-    ])
-  );
-  const status = {
-    rows: 0,
-    totalRows: Number(rows[0].totalRows),
-    totalBytes: Number(bytes[0].totalBytes)
-  };
-  let elapsed = (performance.now() - start).toFixed(3);
-  lc.info?.(`Computed total rows and bytes for: ${select} (${elapsed} ms)`, {
-    status
-  });
-  const copyStream = await tx.processReadTask(
-    (sql) => sql.unsafe(`COPY (${select}) TO STDOUT`).readable()
-  );
-  const tsvParser = new TsvParser();
-  let totalBytes = 0;
-  let totalMsgs = 0;
-  let rowValues = [];
-  let bufferedBytes = 0;
-  const logFlushed = () => {
-    lc.debug?.(
-      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes (total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`
-    );
-  };
-  let row = Array.from({ length: colParsers.length });
-  let col = 0;
-  for await (const data of copyStream) {
-    const chunk = data;
-    for (const text of tsvParser.parse(chunk)) {
-      row[col] = text === null ? null : colParsers[col](text);
-      if (++col === colParsers.length) {
-        rowValues.push(row);
-        status.rows++;
-        row = Array.from({ length: colParsers.length });
-        col = 0;
-      }
-    }
-    bufferedBytes += chunk.byteLength;
-    totalBytes += chunk.byteLength;
-    if (bufferedBytes >= flushThresholdBytes) {
-      yield { tag: "backfill", ...backfill, rowValues, status };
-      totalMsgs++;
-      logFlushed();
-      rowValues = [];
-      bufferedBytes = 0;
-    }
-  }
-  if (rowValues.length > 0) {
-    yield { tag: "backfill", ...backfill, rowValues, status };
-    totalMsgs++;
-    logFlushed();
-  }
-  yield { tag: "backfill-completed", ...backfill, status };
-  elapsed = (performance.now() - start).toFixed(3);
-  lc.info?.(
-    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes (${elapsed} ms)`
-  );
+	const start = performance.now();
+	const [rows, bytes] = await tx.processReadTask((sql) => Promise.all([sql.unsafe(getTotalRows), sql.unsafe(getTotalBytes)]));
+	const status = {
+		rows: 0,
+		totalRows: Number(rows[0].totalRows),
+		totalBytes: Number(bytes[0].totalBytes)
+	};
+	let elapsed = (performance.now() - start).toFixed(3);
+	lc.info?.(`Computed total rows and bytes for: ${select} (${elapsed} ms)`, { status });
+	const copyStream = await tx.processReadTask((sql) => sql.unsafe(`COPY (${select}) TO STDOUT`).readable());
+	const tsvParser = new TsvParser();
+	let totalBytes = 0;
+	let totalMsgs = 0;
+	let rowValues = [];
+	let bufferedBytes = 0;
+	const logFlushed = () => {
+		lc.debug?.(`Flushed ${rowValues.length} rows, ${bufferedBytes} bytes (total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`);
+	};
+	let row = Array.from({ length: colParsers.length });
+	let col = 0;
+	for await (const data of copyStream) {
+		const chunk = data;
+		for (const text of tsvParser.parse(chunk)) {
+			row[col] = text === null ? null : colParsers[col](text);
+			if (++col === colParsers.length) {
+				rowValues.push(row);
+				status.rows++;
+				row = Array.from({ length: colParsers.length });
+				col = 0;
+			}
+		}
+		bufferedBytes += chunk.byteLength;
+		totalBytes += chunk.byteLength;
+		if (bufferedBytes >= flushThresholdBytes) {
+			yield {
+				tag: "backfill",
+				...backfill,
+				rowValues,
+				status
+			};
+			totalMsgs++;
+			logFlushed();
+			rowValues = [];
+			bufferedBytes = 0;
+		}
+	}
+	if (rowValues.length > 0) {
+		yield {
+			tag: "backfill",
+			...backfill,
+			rowValues,
+			status
+		};
+		totalMsgs++;
+		logFlushed();
+	}
+	yield {
+		tag: "backfill-completed",
+		...backfill,
+		status
+	};
+	elapsed = (performance.now() - start).toFixed(3);
+	lc.info?.(`Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes (${elapsed} ms)`);
 }
-async function setSnapshot(lc, upstreamURI, tx, slotNamePrefix) {
-  const replicationSession = pgClient(lc, upstreamURI, {
-    ["fetch_types"]: false,
-    // Necessary for the streaming protocol
-    connection: { replication: "database" }
-    // https://www.postgresql.org/docs/current/protocol-replication.html
-  });
-  const tempSlot = `${slotNamePrefix}_bf_${Date.now()}`;
-  try {
-    const { snapshot_name: snapshot, consistent_point: lsn } = await createReplicationSlot(lc, replicationSession, tempSlot);
-    await tx.processReadTask(
-      (sql) => sql.unsafe(`SET TRANSACTION SNAPSHOT '${snapshot}'`)
-    );
-    await replicationSession.unsafe(`DROP_REPLICATION_SLOT "${tempSlot}"`);
-    const watermark = toStateVersionString(lsn);
-    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);
-    return watermark;
-  } catch (e) {
-    await replicationSession.unsafe(
-      /*sql*/
-      `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots
-         WHERE slot_name = '${tempSlot}'`
-    );
-    lc.error?.(`Failed to create backfill snapshot`, e);
-    throw e;
-  } finally {
-    await replicationSession.end();
-  }
+/**
+* Creates (and drops) a replication slot in order to obtain a snapshot
+* that corresponds with a specific LSN. Sets the snapshot on the
+* TransactionPool and returns the watermark corresponding to the LSN.
+*
+* (Note that PG's other LSN-related functions are not scoped to a
+*  transaction; this is the only way to get set a transaction at a specific
+*  LSN.)
+*/
+async function createSnapshotTransaction(lc, upstreamURI, db, slotNamePrefix) {
+	const replicationSession = pgClient(lc, upstreamURI, {
+		["fetch_types"]: false,
+		connection: { replication: "database" }
+	});
+	const tempSlot = `${slotNamePrefix}_bf_${Date.now()}`;
+	try {
+		const { snapshot_name: snapshot, consistent_point: lsn } = await createReplicationSlot(lc, replicationSession, tempSlot);
+		const { init, imported } = importSnapshot(snapshot);
+		const tx = new TransactionPool(lc, READONLY, init).run(db);
+		await imported;
+		await replicationSession.unsafe(`DROP_REPLICATION_SLOT "${tempSlot}"`);
+		const watermark = toStateVersionString(lsn);
+		lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);
+		return {
+			tx,
+			watermark
+		};
+	} catch (e) {
+		await replicationSession.unsafe(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots
+         WHERE slot_name = '${tempSlot}'`);
+		lc.error?.(`Failed to create backfill snapshot`, e);
+		throw e;
+	} finally {
+		await replicationSession.end();
+	}
 }
 function validateSchema(tx, publications, bf, watermark) {
-  return tx.processReadTask(async (sql) => {
-    const { tables } = await getPublicationInfo(sql, publications);
-    const spec = tables.find(
-      (spec2) => spec2.schema === bf.table.schema && spec2.name === bf.table.name
-    );
-    if (!spec) {
-      throw new SchemaIncompatibilityError(
-        bf,
-        `Table has been renamed or dropped`
-      );
-    }
-    const tableMeta = parse(bf.table.metadata, tableMetadataSchema);
-    if (spec.schemaOID !== tableMeta.schemaOID) {
-      throw new SchemaIncompatibilityError(
-        bf,
-        `Schema no longer corresponds to the original schema`
-      );
-    }
-    if (spec.oid !== tableMeta.relationOID) {
-      throw new SchemaIncompatibilityError(
-        bf,
-        `Table no longer corresponds to the original table`
-      );
-    }
-    if (!equals(
-      new Set(Object.keys(tableMeta.rowKey)),
-      new Set(spec.replicaIdentityColumns)
-    )) {
-      throw new SchemaIncompatibilityError(
-        bf,
-        "Row key (e.g. PRIMARY KEY or INDEX) has changed"
-      );
-    }
-    const allCols = [
-      ...Object.entries(tableMeta.rowKey),
-      ...Object.entries(bf.columns)
-    ];
-    for (const [col, val] of allCols) {
-      const colSpec = spec.columns[col];
-      if (!colSpec) {
-        throw new SchemaIncompatibilityError(
-          bf,
-          `Column ${col} has been renamed or dropped`
-        );
-      }
-      const colMeta = parse(val, columnMetadataSchema);
-      if (colMeta.attNum !== colSpec.pos) {
-        throw new SchemaIncompatibilityError(
-          bf,
-          `Column ${col} no longer corresponds to the original column`
-        );
-      }
-    }
-    const backfill = {
-      relation: {
-        schema: bf.table.schema,
-        name: bf.table.name,
-        rowKey: { columns: Object.keys(tableMeta.rowKey) }
-      },
-      columns: Object.keys(bf.columns).filter(
-        (col) => !(col in tableMeta.rowKey)
-      ),
-      watermark
-    };
-    return { tableSpec: spec, backfill };
-  });
+	return tx.processReadTask(async (sql) => {
+		const { tables } = await getPublicationInfo(sql, publications);
+		const spec = tables.find((spec) => spec.schema === bf.table.schema && spec.name === bf.table.name);
+		if (!spec) throw new SchemaIncompatibilityError(bf, `Table has been renamed or dropped`);
+		const tableMeta = parse(bf.table.metadata, tableMetadataSchema);
+		if (spec.schemaOID !== tableMeta.schemaOID) throw new SchemaIncompatibilityError(bf, `Schema no longer corresponds to the original schema`);
+		if (spec.oid !== tableMeta.relationOID) throw new SchemaIncompatibilityError(bf, `Table no longer corresponds to the original table`);
+		if (!equals(new Set(Object.keys(tableMeta.rowKey)), new Set(spec.replicaIdentityColumns))) throw new SchemaIncompatibilityError(bf, "Row key (e.g. PRIMARY KEY or INDEX) has changed");
+		const allCols = [...Object.entries(tableMeta.rowKey), ...Object.entries(bf.columns)];
+		for (const [col, val] of allCols) {
+			const colSpec = spec.columns[col];
+			if (!colSpec) throw new SchemaIncompatibilityError(bf, `Column ${col} has been renamed or dropped`);
+			if (parse(val, columnMetadataSchema).attNum !== colSpec.pos) throw new SchemaIncompatibilityError(bf, `Column ${col} no longer corresponds to the original column`);
+		}
+		return {
+			tableSpec: spec,
+			backfill: {
+				relation: {
+					schema: bf.table.schema,
+					name: bf.table.name,
+					rowKey: { columns: Object.keys(tableMeta.rowKey) }
+				},
+				columns: Object.keys(bf.columns).filter((col) => !(col in tableMeta.rowKey)),
+				watermark
+			}
+		};
+	});
 }
-export {
-  streamBackfill
-};
-//# sourceMappingURL=backfill-stream.js.map
+//#endregion
+export { streamBackfill };
+
+//# sourceMappingURL=backfill-stream.js.map

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-stream.js","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers, type TypeParser} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {TransactionPool} from '../../../db/transaction-pool.ts';\nimport {pgClient} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  createReplicationSlot,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE} = opts;\n  const db = pgClient(lc, upstreamURI, {\n    connection: {['application_name']: 'backfill-stream'},\n    ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n  });\n  const tx = new TransactionPool(lc, READONLY).run(db);\n  try {\n    const watermark = await setSnapshot(lc, upstreamURI, tx, slot);\n    const {tableSpec, backfill} = await validateSchema(\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n    const types = await getTypeParsers(db, {returnJsonAsString: true});\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n\n    yield* stream(\n      lc,\n      tx,\n      backfill,\n      makeDownloadStatements(tableSpec, cols),\n      cols.map(col => types.getTypeParser(tableSpec.columns[col].typeOID)),\n      flushThresholdBytes,\n    );\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx.setDone();\n    // errors are already thrown and handled from processReadTask()\n    void tx.done().catch(() => {});\n    // Workaround postgres.js hanging at the end of some COPY commands:\n    // https://github.com/porsager/postgres/issues/499\n    void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n  }\n}\n\nasync function* stream(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {select, getTotalRows, getTotalBytes}: DownloadStatements,\n  colParsers: TypeParser[],\n  flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(`Computed total rows and bytes for: ${select} (${elapsed} ms)`, {\n    status,\n  });\n  const copyStream = await tx.processReadTask(sql =>\n    sql.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n  );\n\n  const tsvParser = new TsvParser();\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: colParsers.length});\n  let col = 0;\n\n  for await (const data of copyStream) {\n    const chunk = data as Buffer;\n    for (const text of tsvParser.parse(chunk)) {\n      row[col] = text === null ? null : (colParsers[col](text) as JSONValue);\n\n      if (++col === colParsers.length) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: colParsers.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes) {\n      yield {tag: 'backfill', ...backfill, rowValues, status};\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n  }\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {tag: 'backfill', ...backfill, rowValues, status};\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {tag: 'backfill-completed', ...backfill, status};\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function setSnapshot(\n  lc: LogContext,\n  upstreamURI: string,\n  tx: TransactionPool,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(lc, upstreamURI, {\n    ['fetch_types']: false, // Necessary for the streaming protocol\n    connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n  });\n  const tempSlot = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, tempSlot);\n\n    await tx.processReadTask(sql =>\n      sql.unsafe(`SET TRANSACTION SNAPSHOT '${snapshot}'`),\n    );\n    // Once the snapshot has been set, the replication session and slot can\n    // be closed / dropped.\n    await replicationSession.unsafe(`DROP_REPLICATION_SLOT \"${tempSlot}\"`);\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return watermark;\n  } catch (e) {\n    // In the event of a failure, clean up the replication slot if created.\n    await replicationSession.unsafe(\n      /*sql*/\n      `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n         WHERE slot_name = '${tempSlot}'`,\n    );\n    lc.error?.(`Failed to create backfill snapshot`, e);\n    throw e;\n  } finally {\n    await replicationSession.end();\n  }\n}\n\nfunction validateSchema(\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    return {tableSpec: spec, backfill};\n  });\n}\n"],"names":["spec","v.parse"],"mappings":";;;;;;;;;;;;;;AAgDA,MAAM,2BAA2B,KAAK;AAQtC,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,IAC+B;AACrD,OAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;AAErC,QAAM,EAAC,sBAAsB,yBAAA,IAA4B;AACzD,QAAM,KAAK,SAAS,IAAI,aAAa;AAAA,IACnC,YAAY,EAAC,CAAC,kBAAkB,GAAG,kBAAA;AAAA,IACnC,CAAC,cAAc,GAAG,MAAM;AAAA;AAAA,EAAA,CACzB;AACD,QAAM,KAAK,IAAI,gBAAgB,IAAI,QAAQ,EAAE,IAAI,EAAE;AACnD,MAAI;AACF,UAAM,YAAY,MAAM,YAAY,IAAI,aAAa,IAAI,IAAI;AAC7D,UAAM,EAAC,WAAW,SAAA,IAAY,MAAM;AAAA,MAClC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,UAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK;AAGjE,UAAM,EAAC,UAAU,QAAA,IAAW;AAC5B,UAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;AAEpD,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,uBAAuB,WAAW,IAAI;AAAA,MACtC,KAAK,IAAI,CAAA,QAAO,MAAM,cAAc,UAAU,QAAQ,GAAG,EAAE,OAAO,CAAC;AAAA,MACnE;AAAA,IAAA;AAAA,EAEJ,SAAS,GAAG;AASV,QACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAC7C;AACA,YAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,GAAE;AAAA,IAChE;AACA,UAAM;AAAA,EACR,UAAA;AACE,OAAG,QAAA;AAEH,SAAK,GAAG,OAAO,MAAM,MAAM;AAAA,IAAC,CAAC;AAG7B,SAAK,GAAG,MAAM,MAAM,OAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;AAAA,EAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EAAC,QAAQ,cAAc,cAAA,GACvB,YACA,qBACqD;AACrD,QAAM,QAAQ,YAAY,IAAA;AAC1B,QAAM,CAAC,MAAM,KAAK,IAAI,MAAM,GAAG;AAAA,IAAgB,CAAA,QAC7C,QAAQ,IAAI;AAAA,MACV,IAAI,OAA8B,YAAY;AAAA,MAC9C,IAAI,OAA+B,aAAa;AAAA,IAAA,CACjD;AAAA,EAAA;AAEH,QAAM,SAAyB;AAAA,IAC7B,MAAM;AAAA,IACN,WAAW,OAAO,KAAK,CAAC,EAAE,SAAS;AAAA,IACnC,YAAY,OAAO,MAAM,CAAC,EAAE,UAAU;AAAA,EAAA;AAGxC,MAAI,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACnD,KAAG,OAAO,sCAAsC,MAAM,KAAK,OAAO,QAAQ;AAAA,IACxE;AAAA,EAAA,CACD;AACD,QAAM,aAAa,MAAM,GAAG;AAAA,IAAgB,SAC1C,IAAI,OAAO,SAAS,MAAM,aAAa,EAAE,SAAA;AAAA,EAAS;AAGpD,QAAM,YAAY,IAAI,UAAA;AACtB,MAAI,aAAa;AACjB,MAAI,YAAY;AAChB,MAAI,YAA2B,CAAA;AAC/B,MAAI,gBAAgB;AAEpB,QAAM,aAAa,MAAM;AACvB,OAAG;AAAA,MACD,WAAW,UAAU,MAAM,UAAU,aAAa,uBAChC,OAAO,IAAI,UAAU,SAAS,WAAW,UAAU;AAAA,IAAA;AAAA,EAEzE;AAGA,MAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,WAAW,QAAO;AAC7D,MAAI,MAAM;AAEV,mBAAiB,QAAQ,YAAY;AACnC,UAAM,QAAQ;AACd,eAAW,QAAQ,UAAU,MAAM,KAAK,GAAG;AACzC,UAAI,GAAG,IAAI,SAAS,OAAO,OAAQ,WAAW,GAAG,EAAE,IAAI;AAEvD,UAAI,EAAE,QAAQ,WAAW,QAAQ;AAC/B,kBAAU,KAAK,GAAG;AAClB,eAAO;AACP,cAAM,MAAM,KAAK,EAAC,QAAQ,WAAW,QAAO;AAC5C,cAAM;AAAA,MACR;AAAA,IACF;AACA,qBAAiB,MAAM;AACvB,kBAAc,MAAM;AAEpB,QAAI,iBAAiB,qBAAqB;AACxC,YAAM,EAAC,KAAK,YAAY,GAAG,UAAU,WAAW,OAAA;AAChD;AACA,iBAAA;AACA,kBAAY,CAAA;AACZ,sBAAgB;AAAA,IAClB;AAAA,EACF;AAGA,MAAI,UAAU,SAAS,GAAG;AACxB,UAAM,EAAC,KAAK,YAAY,GAAG,UAAU,WAAW,OAAA;AAChD;AACA,eAAA;AAAA,EACF;AAEA,QAAM,EAAC,KAAK,sBAAsB,GAAG,UAAU,OAAA;AAC/C,aAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AAC/C,KAAG;AAAA,IACD,sBAAsB,OAAO,IAAI,UAAU,SAAS,UAAU,UAAU,WAClE,OAAO;AAAA,EAAA;AAEjB;AAWA,eAAe,YACb,IACA,aACA,IACA,gBACA;AACA,QAAM,qBAAqB,SAAS,IAAI,aAAa;AAAA,IACnD,CAAC,aAAa,GAAG;AAAA;AAAA,IACjB,YAAY,EAAC,aAAa,WAAA;AAAA;AAAA,EAAU,CACrC;AACD,QAAM,WAAW,GAAG,cAAc,OAAO,KAAK,KAAK;AACnD,MAAI;AACF,UAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB,QAAQ;AAE9D,UAAM,GAAG;AAAA,MAAgB,CAAA,QACvB,IAAI,OAAO,6BAA6B,QAAQ,GAAG;AAAA,IAAA;AAIrD,UAAM,mBAAmB,OAAO,0BAA0B,QAAQ,GAAG;AAErE,UAAM,YAAY,qBAAqB,GAAG;AAC1C,OAAG,OAAO,sCAAsC,GAAG,KAAK,SAAS,GAAG;AACpE,WAAO;AAAA,EACT,SAAS,GAAG;AAEV,UAAM,mBAAmB;AAAA;AAAA,MAEvB;AAAA,8BACwB,QAAQ;AAAA,IAAA;AAElC,OAAG,QAAQ,sCAAsC,CAAC;AAClD,UAAM;AAAA,EACR,UAAA;AACE,UAAM,mBAAmB,IAAA;AAAA,EAC3B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;AACD,SAAO,GAAG,gBAAgB,OAAM,QAAO;AACrC,UAAM,EAAC,OAAA,IAAU,MAAM,mBAAmB,KAAK,YAAY;AAC3D,UAAM,OAAO,OAAO;AAAA,MAClB,CAAAA,UAAQA,MAAK,WAAW,GAAG,MAAM,UAAUA,MAAK,SAAS,GAAG,MAAM;AAAA,IAAA;AAEpE,QAAI,CAAC,MAAM;AACT,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,YAAYC,MAAQ,GAAG,MAAM,UAAU,mBAAmB;AAChE,QAAI,KAAK,cAAc,UAAU,WAAW;AAC1C,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,QAAI,KAAK,QAAQ,UAAU,aAAa;AACtC,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,QACE,CAAC;AAAA,MACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC;AAAA,MACrC,IAAI,IAAI,KAAK,sBAAsB;AAAA,IAAA,GAErC;AACA,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,UAAU;AAAA,MACd,GAAG,OAAO,QAAQ,UAAU,MAAM;AAAA,MAClC,GAAG,OAAO,QAAQ,GAAG,OAAO;AAAA,IAAA;AAE9B,eAAW,CAAC,KAAK,GAAG,KAAK,SAAS;AAChC,YAAM,UAAU,KAAK,QAAQ,GAAG;AAChC,UAAI,CAAC,SAAS;AACZ,cAAM,IAAI;AAAA,UACR;AAAA,UACA,UAAU,GAAG;AAAA,QAAA;AAAA,MAEjB;AACA,YAAM,UAAUA,MAAQ,KAAK,oBAAoB;AACjD,UAAI,QAAQ,WAAW,QAAQ,KAAK;AAClC,cAAM,IAAI;AAAA,UACR;AAAA,UACA,UAAU,GAAG;AAAA,QAAA;AAAA,MAEjB;AAAA,IACF;AACA,UAAM,WAA2B;AAAA,MAC/B,UAAU;AAAA,QACR,QAAQ,GAAG,MAAM;AAAA,QACjB,MAAM,GAAG,MAAM;AAAA,QACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAA;AAAA,MAAC;AAAA,MAEjD,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE;AAAA,QAC/B,CAAA,QAAO,EAAE,OAAO,UAAU;AAAA,MAAA;AAAA,MAE5B;AAAA,IAAA;AAEF,WAAO,EAAC,WAAW,MAAM,SAAA;AAAA,EAC3B,CAAC;AACH;"}
+{"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n  PG_UNDEFINED_COLUMN,\n  PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers, type TypeParser} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n  BackfillCompleted,\n  BackfillRequest,\n  DownloadStatus,\n  JSONValue,\n  MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n  columnMetadataSchema,\n  tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n  createReplicationSlot,\n  makeDownloadStatements,\n  type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n  /**\n   * The number of bytes at which to flush a batch of rows in a\n   * backfill message. Defaults to Node's getDefaultHighWatermark().\n   */\n  flushThresholdBytes?: number;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;\n\n/**\n * Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n */\nexport async function* streamBackfill(\n  lc: LogContext,\n  upstreamURI: string,\n  {slot, publications}: Pick<Replica, 'slot' | 'publications'>,\n  bf: BackfillRequest,\n  opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  lc = lc\n    .withContext('component', 'backfill')\n    .withContext('table', bf.table.name);\n\n  const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE} = opts;\n  const db = pgClient(lc, upstreamURI, {\n    connection: {['application_name']: 'backfill-stream'},\n    ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n  });\n  let tx: TransactionPool | undefined;\n  let watermark: string;\n  try {\n    ({tx, watermark} = await createSnapshotTransaction(\n      lc,\n      upstreamURI,\n      db,\n      slot,\n    ));\n    const {tableSpec, backfill} = await validateSchema(\n      tx,\n      publications,\n      bf,\n      watermark,\n    );\n    const types = await getTypeParsers(db, {returnJsonAsString: true});\n\n    // Note: validateSchema ensures that the rowKey and columns are disjoint\n    const {relation, columns} = backfill;\n    const cols = [...relation.rowKey.columns, ...columns];\n\n    yield* stream(\n      lc,\n      tx,\n      backfill,\n      makeDownloadStatements(tableSpec, cols),\n      cols.map(col => types.getTypeParser(tableSpec.columns[col].typeOID)),\n      flushThresholdBytes,\n    );\n  } catch (e) {\n    // Although we make the best effort to validate the schema at the\n    // transaction snapshot, certain forms of `ALTER TABLE` are not\n    // MVCC safe and not \"frozen\" in the snapshot:\n    //\n    // https://www.postgresql.org/docs/current/mvcc-caveats.html\n    //\n    // Handle these errors as schema incompatibility errors rather than\n    // unknown runtime errors.\n    if (\n      e instanceof postgres.PostgresError &&\n      (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)\n    ) {\n      throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n    }\n    throw e;\n  } finally {\n    tx?.setDone();\n    // Workaround postgres.js hanging at the end of some COPY commands:\n    // https://github.com/porsager/postgres/issues/499\n    void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n  }\n}\n\nasync function* stream(\n  lc: LogContext,\n  tx: TransactionPool,\n  backfill: BackfillParams,\n  {select, getTotalRows, getTotalBytes}: DownloadStatements,\n  colParsers: TypeParser[],\n  flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill | BackfillCompleted> {\n  const start = performance.now();\n  const [rows, bytes] = await tx.processReadTask(sql =>\n    Promise.all([\n      sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n      sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n    ]),\n  );\n  const status: DownloadStatus = {\n    rows: 0,\n    totalRows: Number(rows[0].totalRows),\n    totalBytes: Number(bytes[0].totalBytes),\n  };\n\n  let elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(`Computed total rows and bytes for: ${select} (${elapsed} ms)`, {\n    status,\n  });\n  const copyStream = await tx.processReadTask(sql =>\n    sql.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n  );\n\n  const tsvParser = new TsvParser();\n  let totalBytes = 0;\n  let totalMsgs = 0;\n  let rowValues: JSONValue[][] = [];\n  let bufferedBytes = 0;\n\n  const logFlushed = () => {\n    lc.debug?.(\n      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n        `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n    );\n  };\n\n  // Tracks the row being parsed.\n  let row: JSONValue[] = Array.from({length: colParsers.length});\n  let col = 0;\n\n  for await (const data of copyStream) {\n    const chunk = data as Buffer;\n    for (const text of tsvParser.parse(chunk)) {\n      row[col] = text === null ? null : (colParsers[col](text) as JSONValue);\n\n      if (++col === colParsers.length) {\n        rowValues.push(row);\n        status.rows++;\n        row = Array.from({length: colParsers.length});\n        col = 0;\n      }\n    }\n    bufferedBytes += chunk.byteLength;\n    totalBytes += chunk.byteLength;\n\n    if (bufferedBytes >= flushThresholdBytes) {\n      yield {tag: 'backfill', ...backfill, rowValues, status};\n      totalMsgs++;\n      logFlushed();\n      rowValues = [];\n      bufferedBytes = 0;\n    }\n  }\n\n  // Flush the last batch of rows.\n  if (rowValues.length > 0) {\n    yield {tag: 'backfill', ...backfill, rowValues, status};\n    totalMsgs++;\n    logFlushed();\n  }\n\n  yield {tag: 'backfill-completed', ...backfill, status};\n  elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(\n    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n      `(${elapsed} ms)`,\n  );\n}\n\n/**\n * Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n *\n * (Note that PG's other LSN-related functions are not scoped to a\n *  transaction; this is the only way to get set a transaction at a specific\n *  LSN.)\n */\nasync function createSnapshotTransaction(\n  lc: LogContext,\n  upstreamURI: string,\n  db: PostgresDB,\n  slotNamePrefix: string,\n) {\n  const replicationSession = pgClient(lc, upstreamURI, {\n    ['fetch_types']: false, // Necessary for the streaming protocol\n    connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n  });\n  const tempSlot = `${slotNamePrefix}_bf_${Date.now()}`;\n  try {\n    const {snapshot_name: snapshot, consistent_point: lsn} =\n      await createReplicationSlot(lc, replicationSession, tempSlot);\n\n    const {init, imported} = importSnapshot(snapshot);\n    const tx = new TransactionPool(lc, READONLY, init).run(db);\n    await imported;\n    await replicationSession.unsafe(`DROP_REPLICATION_SLOT \"${tempSlot}\"`);\n\n    const watermark = toStateVersionString(lsn);\n    lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n    return {tx, watermark};\n  } catch (e) {\n    // In the event of a failure, clean up the replication slot if created.\n    await replicationSession.unsafe(\n      /*sql*/\n      `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n         WHERE slot_name = '${tempSlot}'`,\n    );\n    lc.error?.(`Failed to create backfill snapshot`, e);\n    throw e;\n  } finally {\n    await replicationSession.end();\n  }\n}\n\nfunction validateSchema(\n  tx: TransactionPool,\n  publications: string[],\n  bf: BackfillRequest,\n  watermark: string,\n): Promise<{\n  tableSpec: PublishedTableSpec;\n  backfill: BackfillParams;\n}> {\n  return tx.processReadTask(async sql => {\n    const {tables} = await getPublicationInfo(sql, publications);\n    const spec = tables.find(\n      spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n    );\n    if (!spec) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table has been renamed or dropped`,\n      );\n    }\n    const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n    if (spec.schemaOID !== tableMeta.schemaOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Schema no longer corresponds to the original schema`,\n      );\n    }\n    if (spec.oid !== tableMeta.relationOID) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        `Table no longer corresponds to the original table`,\n      );\n    }\n    if (\n      !equals(\n        new Set(Object.keys(tableMeta.rowKey)),\n        new Set(spec.replicaIdentityColumns),\n      )\n    ) {\n      throw new SchemaIncompatibilityError(\n        bf,\n        'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n      );\n    }\n    const allCols = [\n      ...Object.entries(tableMeta.rowKey),\n      ...Object.entries(bf.columns),\n    ];\n    for (const [col, val] of allCols) {\n      const colSpec = spec.columns[col];\n      if (!colSpec) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} has been renamed or dropped`,\n        );\n      }\n      const colMeta = v.parse(val, columnMetadataSchema);\n      if (colMeta.attNum !== colSpec.pos) {\n        throw new SchemaIncompatibilityError(\n          bf,\n          `Column ${col} no longer corresponds to the original column`,\n        );\n      }\n    }\n    const backfill: BackfillParams = {\n      relation: {\n        schema: bf.table.schema,\n        name: bf.table.name,\n        rowKey: {columns: Object.keys(tableMeta.rowKey)},\n      },\n      columns: Object.keys(bf.columns).filter(\n        col => !(col in tableMeta.rowKey),\n      ),\n      watermark,\n    };\n    return {tableSpec: spec, backfill};\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAgDA,IAAM,2BAA2B,KAAK;;;;;;;AAQtC,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,EAAE,EAC6B;AACrD,MAAK,GACF,YAAY,aAAa,WAAW,CACpC,YAAY,SAAS,GAAG,MAAM,KAAK;CAEtC,MAAM,EAAC,sBAAsB,6BAA4B;CACzD,MAAM,KAAK,SAAS,IAAI,aAAa;EACnC,YAAY,GAAE,qBAAqB,mBAAkB;GACpD,iBAAiB;EACnB,CAAC;CACF,IAAI;CACJ,IAAI;AACJ,KAAI;AACF,GAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,KACD;EACD,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,UACD;EACD,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK,CAAC;EAGlE,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,QAAQ;AAErD,SAAO,OACL,IACA,IACA,UACA,uBAAuB,WAAW,KAAK,EACvC,KAAK,KAAI,QAAO,MAAM,cAAc,UAAU,QAAQ,KAAK,QAAQ,CAAC,EACpE,oBACD;UACM,GAAG;AASV,MACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,qBAE7C,OAAM,IAAI,2BAA2B,IAAI,OAAO,EAAE,EAAE,EAAC,OAAO,GAAE,CAAC;AAEjE,QAAM;WACE;AACR,MAAI,SAAS;AAGR,KAAG,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,qCAAqC,EAAE,CAAC;;;AAI/E,gBAAgB,OACd,IACA,IACA,UACA,EAAC,QAAQ,cAAc,iBACvB,YACA,qBACqD;CACrD,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,aAAa,EAC/C,IAAI,OAA+B,cAAc,CAClD,CAAC,CACH;CACD,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,UAAU;EACpC,YAAY,OAAO,MAAM,GAAG,WAAW;EACxC;CAED,IAAI,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACpD,IAAG,OAAO,sCAAsC,OAAO,IAAI,QAAQ,OAAO,EACxE,QACD,CAAC;CACF,MAAM,aAAa,MAAM,GAAG,iBAAgB,QAC1C,IAAI,OAAO,SAAS,OAAO,aAAa,CAAC,UAAU,CACpD;CAED,MAAM,YAAY,IAAI,WAAW;CACjC,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,EAAE;CACjC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;AACvB,KAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,GACvE;;CAIH,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,WAAW,QAAO,CAAC;CAC9D,IAAI,MAAM;AAEV,YAAW,MAAM,QAAQ,YAAY;EACnC,MAAM,QAAQ;AACd,OAAK,MAAM,QAAQ,UAAU,MAAM,MAAM,EAAE;AACzC,OAAI,OAAO,SAAS,OAAO,OAAQ,WAAW,KAAK,KAAK;AAExD,OAAI,EAAE,QAAQ,WAAW,QAAQ;AAC/B,cAAU,KAAK,IAAI;AACnB,WAAO;AACP,UAAM,MAAM,KAAK,EAAC,QAAQ,WAAW,QAAO,CAAC;AAC7C,UAAM;;;AAGV,mBAAiB,MAAM;AACvB,gBAAc,MAAM;AAEpB,MAAI,iBAAiB,qBAAqB;AACxC,SAAM;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;IAAO;AACvD;AACA,eAAY;AACZ,eAAY,EAAE;AACd,mBAAgB;;;AAKpB,KAAI,UAAU,SAAS,GAAG;AACxB,QAAM;GAAC,KAAK;GAAY,GAAG;GAAU;GAAW;GAAO;AACvD;AACA,cAAY;;AAGd,OAAM;EAAC,KAAK;EAAsB,GAAG;EAAU;EAAO;AACtD,YAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AAChD,IAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,MACf;;;;;;;;;;;AAYH,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SAAS,IAAI,aAAa;GAClD,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CAAC;CACF,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,KAAK;AACnD,KAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB,SAAS;EAE/D,MAAM,EAAC,MAAM,aAAY,eAAe,SAAS;EACjD,MAAM,KAAK,IAAI,gBAAgB,IAAI,UAAU,KAAK,CAAC,IAAI,GAAG;AAC1D,QAAM;AACN,QAAM,mBAAmB,OAAO,0BAA0B,SAAS,GAAG;EAEtE,MAAM,YAAY,qBAAqB,IAAI;AAC3C,KAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,GAAG;AACrE,SAAO;GAAC;GAAI;GAAU;UACf,GAAG;AAEV,QAAM,mBAAmB,OAEvB;8BACwB,SAAS,GAClC;AACD,KAAG,QAAQ,sCAAsC,EAAE;AACnD,QAAM;WACE;AACR,QAAM,mBAAmB,KAAK;;;AAIlC,SAAS,eACP,IACA,cACA,IACA,WAIC;AACD,QAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,aAAa;EAC5D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,KACnE;AACD,MAAI,CAAC,KACH,OAAM,IAAI,2BACR,IACA,oCACD;EAEH,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,oBAAoB;AACjE,MAAI,KAAK,cAAc,UAAU,UAC/B,OAAM,IAAI,2BACR,IACA,sDACD;AAEH,MAAI,KAAK,QAAQ,UAAU,YACzB,OAAM,IAAI,2BACR,IACA,oDACD;AAEH,MACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,OAAO,CAAC,EACtC,IAAI,IAAI,KAAK,uBAAuB,CACrC,CAED,OAAM,IAAI,2BACR,IACA,kDACD;EAEH,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,OAAO,EACnC,GAAG,OAAO,QAAQ,GAAG,QAAQ,CAC9B;AACD,OAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;AAC7B,OAAI,CAAC,QACH,OAAM,IAAI,2BACR,IACA,UAAU,IAAI,8BACf;AAGH,OADgB,MAAQ,KAAK,qBAAqB,CACtC,WAAW,QAAQ,IAC7B,OAAM,IAAI,2BACR,IACA,UAAU,IAAI,+CACf;;AAcL,SAAO;GAAC,WAAW;GAAM,UAXQ;IAC/B,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,OAAO,EAAC;KACjD;IACD,SAAS,OAAO,KAAK,GAAG,QAAQ,CAAC,QAC/B,QAAO,EAAE,OAAO,UAAU,QAC3B;IACD;IACD;GACiC;GAClC"}

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAqBjD,OAAO,KAAK,EAAa,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAEzE,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAUhD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AAuBjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAqC7E;AA8YD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AA0gBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAqBjD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAUhD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AAuBjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAqC7E;AA8YD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAgiBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}