npm - @rocicorp/zero - Versions diffs - 1.11.0-canary.13 → 1.11.0-canary.15 - Mend

@rocicorp/zero 1.11.0-canary.13 → 1.11.0-canary.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (252) hide show

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js CHANGED Viewed

@@ -1,11 +1,15 @@
 import { assert } from "../../../../../shared/src/asserts.js";
 import { parse } from "../../../../../shared/src/valita.js";
+import { must } from "../../../../../shared/src/must.js";
+import { randomCharacters } from "../../../../../shared/src/random-values.js";
 import { equals } from "../../../../../shared/src/set-utils.js";
+import { id } from "../../../types/sql.js";
 import { connectPgClient, pgClient } from "../../../types/pg.js";
 import { toStateVersionString } from "./lsn.js";
 import { READONLY } from "../../../db/mode-enum.js";
 import { getPublicationInfo } from "./schema/published.js";
 import { Queue } from "../../../../../shared/src/queue.js";
+import { resumePoint } from "../protocol/backfill-progress.js";
 import { SchemaIncompatibilityError } from "../common/backfill-manager.js";
 import { BinaryCopyParser, hasBinaryDecoder, makeBinaryDecoder, textCastDecoder } from "../../../db/pg-copy-binary.js";
 import { TsvParser } from "../../../db/pg-copy.js";
@@ -17,6 +21,52 @@ import { makeBinarySelectExprs, makeDownloadStatements } from "./initial-sync.js
 import postgres from "postgres";
 import { PG_UNDEFINED_COLUMN, PG_UNDEFINED_TABLE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/backfill-stream.ts
+/**
+* Encodes a `ctid` text representation, e.g. `(123,4)`, as a
+* fixed-width string that sorts lexicographically in `ctid` order.
+*/
+var CTID_RE = /^\((\d+),(\d+)\)$/;
+var CTID_MARK_RE = /^(\d{10})\.(\d{5})$/;
+function ctidToProgressMark(ctid) {
+	const match = CTID_RE.exec(ctid);
+	assert(match, () => `invalid ctid: ${ctid}`);
+	return `${match[1].padStart(10, "0")}.${match[2].padStart(5, "0")}`;
+}
+/** The inverse of {@link ctidToProgressMark}, as a `tid` literal. */
+function progressMarkToCtid(mark) {
+	const match = CTID_MARK_RE.exec(mark);
+	assert(match, () => `invalid ctid progress mark: ${mark}`);
+	return `(${Number(match[1])},${Number(match[2])})`;
+}
+function progressMarks(previous, current) {
+	return previous ? {
+		previous,
+		current
+	} : { current };
+}
+/** The `ctid` of each row is appended as the last column of the download. */
+var CTID_SELECT = "ctid::text";
+/**
+* Settings that ensure that the COPY scans the table in physical (i.e.
+* `ctid`) order, which is necessary for the progress marks of a resumable
+* backfill to be meaningful. (Monotonicity is nonetheless verified as rows
+* are streamed.)
+*
+* * Synchronized seqscans can start in the middle of the table.
+* * Parallel scans interleave the rows of different workers.
+* * Index scans follow index order. (A publication row filter could
+*   otherwise be satisfied with an index.)
+*/
+var PHYSICAL_ORDER_SETTINGS = [
+	"synchronize_seqscans = off",
+	"max_parallel_workers_per_gather = 0",
+	"enable_indexscan = off",
+	"enable_indexonlyscan = off",
+	"enable_bitmapscan = off"
+];
+function rowCountMark(rows) {
+	return String(rows).padStart(16, "0");
+}
 var POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;
 var SAMPLE_OR_LIMIT_RE = /\sTABLESAMPLE\s+BERNOULLI\b|\sLIMIT\s+\d/i;
 /**
@@ -27,7 +77,7 @@ var SAMPLE_OR_LIMIT_RE = /\sTABLESAMPLE\s+BERNOULLI\b|\sLIMIT\s+\d/i;
 */
 async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts = {}) {
 	lc = lc.withContext("component", "backfill").withContext("table", bf.table.name);
-	const { flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false } = opts;
+	const { flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false, afterSnapshotForTesting } = opts;
 	const db = await connectPgClient(lc, upstreamURI, "backfill-stream", {
 		["max_lifetime"]: 1440 * 60,
 		liveness: "keepalive"
@@ -36,23 +86,26 @@ async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts
 	let watermark;
 	try {
 		({tx, watermark} = await createSnapshotTransaction(lc, upstreamURI, db, slot));
-		const { tableSpec, backfill } = await validateSchema(tx, publications, bf, watermark);
+		await afterSnapshotForTesting?.();
+		const { tableSpec, backfill, progress } = await validateSchema(lc, tx, publications, bf, watermark);
 		const { relation, columns } = backfill;
 		const cols = [...relation.rowKey.columns, ...columns];
-		const stmts = makeDownloadStatements(tableSpec, cols);
+		const order = progress.resumable && progress.start ? { after: `ctid > '${progressMarkToCtid(progress.start.progressMark)}'::tid` } : void 0;
+		const stmts = makeDownloadStatements(tableSpec, cols, void 0, void 0, [...cols.map((col) => id(col)), CTID_SELECT], order);
+		if (progress.start) lc.info?.(`resuming backfill from ${progress.start.progressMark} (timeline ${progress.timeline})`);
 		if (textCopy) {
 			const types = await getTypeParsers(db, { returnJsonAsString: true });
-			yield* stream(lc, tx, backfill, stmts, `COPY (${stmts.select}) TO STDOUT`, new TsvParser(), cols.map((col) => {
+			yield* stream(lc, tx, backfill, progress, stmts, `COPY (${stmts.select}) TO STDOUT`, new TsvParser(), cols.map((col) => {
 				const parser = types.getTypeParser(tableSpec.columns[col].typeOID);
 				return (text) => parser(text);
-			}), flushThresholdBytes);
+			}), (text) => text, flushThresholdBytes);
 		} else {
-			const binaryStmts = makeDownloadStatements(tableSpec, cols, void 0, void 0, makeBinarySelectExprs(tableSpec, cols));
-			yield* stream(lc, tx, backfill, stmts, `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`, new BinaryCopyParser(), cols.map((col) => {
+			const binaryStmts = makeDownloadStatements(tableSpec, cols, void 0, void 0, [...makeBinarySelectExprs(tableSpec, cols), CTID_SELECT], order);
+			yield* stream(lc, tx, backfill, progress, stmts, `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`, new BinaryCopyParser(), cols.map((col) => {
 				const spec = tableSpec.columns[col];
 				const decoder = hasBinaryDecoder(spec) ? makeBinaryDecoder(spec) : textCastDecoder;
 				return (buf) => decoder(buf);
-			}), flushThresholdBytes);
+			}), (buf) => buf.toString("utf8"), flushThresholdBytes);
 		}
 	} catch (e) {
 		if (e instanceof postgres.PostgresError && (e.code === PG_UNDEFINED_TABLE || e.code === PG_UNDEFINED_COLUMN)) throw new SchemaIncompatibilityError(bf, String(e), { cause: e });
@@ -62,7 +115,7 @@ async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts
 		db.end().catch((e) => lc.warn?.(`error closing backfill connection`, e));
 	}
 }
-async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCommand, parser, decoders, flushThresholdBytes) {
+async function* stream(lc, tx, backfill, { timeline, resumable, start: startMark }, { getTotalRows, getTotalBytes }, copyCommand, parser, decoders, ctidDecoder, flushThresholdBytes) {
 	assert(!SAMPLE_OR_LIMIT_RE.test(copyCommand), `backfill COPY must not sample or limit: ${copyCommand}`);
 	const start = performance.now();
 	const [rows, bytes] = await tx.processReadTask((sql) => Promise.all([sql.unsafe(getTotalRows), sql.unsafe(getTotalBytes)]));
@@ -76,6 +129,7 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 	const chunks = new Queue();
 	const acks = new Queue();
 	const copyDone = tx.processReadTask(async (sql) => {
+		for (const setting of PHYSICAL_ORDER_SETTINGS) await sql.unsafe(`SET LOCAL ${setting}`);
 		const readable = await sql.unsafe(copyCommand).readable();
 		for await (const chunk of readable) {
 			chunks.enqueue(chunk);
@@ -91,14 +145,26 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 	const logFlushed = () => {
 		lc.debug?.(`Flushed ${rowValues.length} rows, ${bufferedBytes} bytes (total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`);
 	};
+	let current = startMark;
+	let previous = startMark;
+	const markOf = (progressMark) => ({
+		progressMark,
+		timeline
+	});
+	const numCols = decoders.length + 1;
 	let row = Array.from({ length: decoders.length });
 	let col = 0;
 	for (;;) {
 		const chunk = await chunks.dequeue();
 		if (chunk === "done") break;
 		for (const field of parser.parse(chunk)) {
-			row[col] = field === null ? null : decoders[col](field);
-			if (++col === decoders.length) {
+			if (col < decoders.length) row[col] = field === null ? null : decoders[col](field);
+			else {
+				const mark = resumable ? ctidToProgressMark(ctidDecoder(field)) : rowCountMark(status.rows + 1);
+				if (current !== void 0 && mark <= current.progressMark) throw new Error(`backfill rows are not in ctid order (${mark} after ${current.progressMark})`);
+				current = markOf(mark);
+			}
+			if (++col === numCols) {
 				rowValues.push(row);
 				status.rows++;
 				row = Array.from({ length: decoders.length });
@@ -107,16 +173,18 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 		}
 		bufferedBytes += chunk.byteLength;
 		totalBytes += chunk.byteLength;
-		if (bufferedBytes >= flushThresholdBytes) {
+		if (bufferedBytes >= flushThresholdBytes && rowValues.length > 0) {
 			yield {
 				message: {
 					tag: "backfill",
 					...backfill,
 					rowValues,
-					status
+					status,
+					progressMarks: progressMarks(previous, must(current))
 				},
 				byteSize: bufferedBytes
 			};
+			previous = current;
 			totalMsgs++;
 			logFlushed();
 			rowValues = [];
@@ -131,10 +199,12 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 				tag: "backfill",
 				...backfill,
 				rowValues,
-				status
+				status,
+				progressMarks: progressMarks(previous, must(current))
 			},
 			byteSize: bufferedBytes
 		};
+		previous = current;
 		totalMsgs++;
 		logFlushed();
 	}
@@ -142,7 +212,8 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 		message: {
 			tag: "backfill-completed",
 			...backfill,
-			status
+			status,
+			progressMarks: previous ? { previous } : {}
 		},
 		byteSize: 0
 	};
@@ -185,7 +256,7 @@ async function createSnapshotTransaction(lc, upstreamURI, db, slotNamePrefix) {
 		await replicationSession.end();
 	}
 }
-function validateSchema(tx, publications, bf, watermark) {
+function validateSchema(lc, tx, publications, bf, watermark) {
 	return tx.processReadTask(async (sql) => {
 		const { tables } = await getPublicationInfo(sql, publications);
 		const spec = tables.find((spec) => spec.schema === bf.table.schema && spec.name === bf.table.name);
@@ -194,7 +265,7 @@ function validateSchema(tx, publications, bf, watermark) {
 		if (spec.schemaOID !== tableMeta.schemaOID) throw new SchemaIncompatibilityError(bf, `Schema no longer corresponds to the original schema`);
 		if (spec.oid !== tableMeta.relationOID) throw new SchemaIncompatibilityError(bf, `Table no longer corresponds to the original table`);
 		if (!equals(new Set(Object.keys(tableMeta.rowKey)), new Set(spec.replicaIdentityColumns))) throw new SchemaIncompatibilityError(bf, "Row key (e.g. PRIMARY KEY or INDEX) has changed");
-		const allCols = [...Object.entries(tableMeta.rowKey), ...Object.entries(bf.columns)];
+		const allCols = [...Object.entries(tableMeta.rowKey), ...Object.entries(bf.columns).map(([col, { id }]) => [col, id])];
 		for (const [col, val] of allCols) {
 			const colSpec = spec.columns[col];
 			if (!colSpec) throw new SchemaIncompatibilityError(bf, `Column ${col} has been renamed or dropped`);
@@ -210,10 +281,125 @@ function validateSchema(tx, publications, bf, watermark) {
 				},
 				columns: Object.keys(bf.columns).filter((col) => !(col in tableMeta.rowKey)),
 				watermark
-			}
+			},
+			progress: await getProgress(lc, sql, spec.oid, bf)
 		};
 	});
 }
+/**
+* Determines the timeline of the table's progress marks (as of the snapshot),
+* and whether the backfill can be resumed from the requested progress.
+*
+* The table is locked (in ACCESS SHARE mode, as the COPY would, for the rest
+* of the transaction) so that its storage cannot be rewritten (by `VACUUM
+* FULL`, `CLUSTER`, `TRUNCATE`, or a rewriting `ALTER TABLE`, all of which
+* require ACCESS EXCLUSIVE) between reading the timeline and the COPY. Note
+* that `pg_relation_filenode()` reflects the table's current storage (only
+* refreshed upon acquiring a lock), whereas `pg_class.relfilenode` is read at
+* the transaction snapshot. If they differ, the storage was rewritten after the
+* snapshot, and the COPY would scan the new storage with the old snapshot,
+* which is not MVCC-safe for all rewrites (e.g. the table appears empty after
+* a rewriting `ALTER TABLE`). The backfill is then retried at a new snapshot.
+*/
+async function getProgress(lc, sql, relationOID, bf) {
+	const table = `${id(bf.table.schema)}.${id(bf.table.name)}`;
+	await sql.unsafe(`LOCK TABLE ${table} IN ACCESS SHARE MODE`);
+	const [{ lockedOID, relkind, snapshotFilenode, filenode }] = await sql`
+    SELECT to_regclass(${table})::oid::int8 AS "lockedOID",
+           relkind,
+           relfilenode::text AS "snapshotFilenode",
+           pg_relation_filenode(oid)::text AS filenode
+      FROM pg_class WHERE oid = ${relationOID}`;
+	if (Number(lockedOID) !== relationOID) throw new SchemaIncompatibilityError(bf, `Table has been renamed or replaced`);
+	if (relkind !== "r" || filenode === null) return {
+		timeline: `${relkind}:${randomCharacters(12)}`,
+		resumable: false,
+		start: void 0
+	};
+	if (filenode !== snapshotFilenode) throw new Error(`${bf.table.schema}.${bf.table.name} was rewritten after the backfill snapshot (relfilenode ${snapshotFilenode} => ${filenode})`);
+	const timeline = filenode;
+	const requested = resumePoint(bf);
+	if (requested && requested.timeline !== timeline) {
+		lc.info?.(`restarting backfill from scratch: the requested progress is on timeline ${requested.timeline} but the table is on ${timeline}`);
+		return {
+			timeline,
+			resumable: true,
+			start: void 0
+		};
+	}
+	if (requested && !await canResume(sql, relationOID, bf)) {
+		lc.info?.(`restarting backfill from scratch: the table has TOASTed values and backfilling columns [${Object.keys(bf.columns).join(",")}] are TOAST-able`);
+		return {
+			timeline,
+			resumable: true,
+			start: void 0
+		};
+	}
+	return {
+		timeline,
+		resumable: true,
+		start: requested
+	};
+}
+/**
+* Whether resuming a backfill (in a new snapshot) is guaranteed to deliver
+* the values of every row that the preceding snapshot(s) had not reached.
+*
+* Within a single snapshot, every row is scanned exactly once, since the
+* version visible to the snapshot stays in place while it is held. Across
+* snapshots, however, a row that had not yet been reached (i.e. beyond the
+* resume mark `M`) can be updated between the snapshots, with its new
+* version placed at or before `M` (a non-HOT update into free space in an
+* earlier page, or a HOT update into a lower line pointer of `M`'s page).
+* Neither snapshot then scans the row, and it is only delivered by the
+* replication stream. The stream's UPDATE contains the full row, _except_
+* for unchanged values stored out-of-line in TOAST, which pgoutput omits
+* (unless the table has REPLICA IDENTITY FULL). A backfilling column with
+* such a value would be published without it.
+*
+* Resuming is therefore only done if no backfilling column can be TOASTed
+* (i.e. all have `attstorage = 'p'`, as for fixed-width types), or if the
+* table has no TOAST data. The latter is safe because an omitted value must
+* still be in the TOAST relation at the new snapshot (unless its row has since
+* been deleted, which the replication stream does deliver), and conservative
+* because the TOAST relation does not shrink to empty without a heap rewrite,
+* which changes the timeline.
+*
+* Potential optimization for TOAST-able backfills: have the resumed run also
+* deliver the rows at or before `M` that were modified since the snapshot of
+* the run that started the backfill from scratch. That run's snapshot `xmin`
+* (`T_origin`, as an xid8) would be encoded in the timeline, i.e.
+* `timeline = "<relfilenode>:<T_origin>"`, and carried over (rather than
+* replaced) by resumed runs, so that:
+* * Within a chain of runs, every row version with `xmin < T_origin` stayed
+*   in place across all of the runs' snapshots and was therefore scanned
+*   exactly once, and all other rows are re-delivered by the resumed run
+*   (`ctid > M OR xmin >= T_origin`).
+* * Continuations from a different chain (i.e. with a newer `T_origin`) are
+*   not accepted by subscribers, and requests from different chains are
+*   merged into a backfill from scratch, as the timelines differ. Note that
+*   using the _latest_ `T` would be incorrect: rows that moved before `M`
+*   between the earlier snapshots would never be re-delivered.
+* The resumed run would scan the whole table (but only deliver the modified
+* rows), comparing the 32-bit `xmin` with `T_origin` modulo 2^32, which
+* requires falling back to a backfill from scratch once the current xid is
+* 2^31 or more past `T_origin` (i.e. wraparound). This also relies on the
+* `xmin` system column returning the raw xmin of frozen tuples, which should
+* be verified.
+*/
+async function canResume(sql, relationOID, bf) {
+	const [{ toastable, toasted }] = await sql`
+    SELECT
+      EXISTS (
+        SELECT 1 FROM pg_attribute
+          WHERE attrelid = ${relationOID}
+            AND attname IN ${sql(Object.keys(bf.columns))}
+            AND attstorage <> 'p'
+      ) AS toastable,
+      reltoastrelid <> 0 AND pg_relation_size(reltoastrelid) > 0 AS toasted
+      FROM pg_class WHERE oid = ${relationOID}`;
+	return !toastable || !toasted;
+}
 //#endregion
 export { streamBackfill };

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n PG_UNDEFINED_COLUMN,\n PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {\n SchemaIncompatibilityError,\n type BackfillMessage,\n} from '../common/backfill-manager.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n DownloadStatus,\n JSONValue,\n} from '../protocol/current.ts';\nimport {\n columnMetadataSchema,\n tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n makeBinarySelectExprs,\n makeDownloadStatements,\n type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n /*\n The number of bytes at which to flush a batch of rows in a\n * backfill message. Defaults to Node's getDefaultHighWatermark().\n /\n flushThresholdBytes?: number \| undefined;\n\n /\n Use text-format COPY instead of binary COPY.\n * Binary is faster and handles all types (unknown types are cast to\n * `::text` in the SELECT). This flag exists as an escape hatch to\n * revert to the old code path if needed.\n /\n textCopy?: boolean \| undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b\|\\sLIMIT\\s+\\d/i;\n\n/*\n Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n /\nexport async function streamBackfill(\n lc: LogContext,\n upstreamURI: string,\n {slot, publications}: Pick<Replica, 'slot' \| 'publications'>,\n bf: BackfillRequest,\n opts: StreamOptions = {},\n): AsyncGenerator<BackfillMessage> {\n lc = lc\n .withContext('component', 'backfill')\n .withContext('table', bf.table.name);\n\n const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false} =\n opts;\n const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n // The COPY is a single stream that must outlive the entire table download,\n // so allow a very long (24h) connection lifetime.\n ['max_lifetime']: 24 * 60 * 60,\n // A backfill COPY is a background process that can be preempted by\n // upstream replication changes, potentially requiring it to sit idle for\n // long stretches. Use TCP keepalive to detect dead connections instead\n // of the inactivity watchdog, which could kill connnections that are\n // taking a back seat for upstream replication changes.\n liveness: 'keepalive',\n });\n let tx: TransactionPool \| undefined;\n let watermark: string;\n try {\n ({tx, watermark} = await createSnapshotTransaction(\n lc,\n upstreamURI,\n db,\n slot,\n ));\n const {tableSpec, backfill} = await validateSchema(\n tx,\n publications,\n bf,\n watermark,\n );\n\n // Note: validateSchema ensures that the rowKey and columns are disjoint\n const {relation, columns} = backfill;\n const cols = [...relation.rowKey.columns, ...columns];\n const stmts = makeDownloadStatements(tableSpec, cols);\n\n if (textCopy) {\n const types = await getTypeParsers(db, {returnJsonAsString: true});\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${stmts.select}) TO STDOUT`,\n new TsvParser(),\n cols.map(col => {\n const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n return (text: string) => parser(text) as JSONValue;\n }),\n flushThresholdBytes,\n );\n } else {\n const binaryStmts = makeDownloadStatements(\n tableSpec,\n cols,\n undefined,\n undefined,\n makeBinarySelectExprs(tableSpec, cols),\n );\n\n yield* stream(\n lc,\n tx,\n backfill,\n stmts,\n `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n new BinaryCopyParser(),\n cols.map(col => {\n const spec = tableSpec.columns[col];\n const decoder = hasBinaryDecoder(spec)\n ? makeBinaryDecoder(spec)\n : textCastDecoder;\n return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n }),\n flushThresholdBytes,\n );\n }\n } catch (e) {\n // Although we make the best effort to validate the schema at the\n // transaction snapshot, certain forms of `ALTER TABLE` are not\n // MVCC safe and not \"frozen\" in the snapshot:\n //\n // https://www.postgresql.org/docs/current/mvcc-caveats.html\n //\n // Handle these errors as schema incompatibility errors rather than\n // unknown runtime errors.\n if (\n e instanceof postgres.PostgresError &&\n (e.code === PG_UNDEFINED_TABLE \|\| e.code === PG_UNDEFINED_COLUMN)\n ) {\n throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n }\n throw e;\n } finally {\n tx?.setDone();\n // Workaround postgres.js hanging at the end of some COPY commands:\n // https://github.com/porsager/postgres/issues/499\n void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n }\n}\n\nasync function* stream<T>(\n lc: LogContext,\n tx: TransactionPool,\n backfill: BackfillParams,\n {\n getTotalRows,\n getTotalBytes,\n }: Pick<DownloadStatements, 'getTotalRows' \| 'getTotalBytes'>,\n copyCommand: string,\n parser: {parse(chunk: Buffer): Iterable<T \| null>},\n decoders: ((field: T) => JSONValue)[],\n flushThresholdBytes: number,\n): AsyncGenerator<BackfillMessage> {\n // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n // sync and must never appear in a backfill COPY.\n assert(\n !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n `backfill COPY must not sample or limit: ${copyCommand}`,\n );\n const start = performance.now();\n const [rows, bytes] = await tx.processReadTask(sql =>\n Promise.all([\n sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n ]),\n );\n const status: DownloadStatus = {\n rows: 0,\n totalRows: Number(rows[0].totalRows),\n totalBytes: Number(bytes[0].totalBytes),\n };\n\n let elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n {\n status,\n },\n );\n\n // Drain the COPY stream from within a single read task so that the\n // TransactionPool worker holds the transaction for the entire duration of\n // the COPY, rather than incorrectly considering the worker \"idle\" and\n // attempting to send keepalives on it.\n //\n // Chunks are bridged out to this generator via a Queue, with a corresponding\n // acks Queue providing strict one-chunk-at-a-time backpressure. This is\n // necessary because it is not possible to \"yield\" from within the read task.\n const chunks = new Queue<Buffer \| 'done'>();\n const acks = new Queue<void>();\n\n const copyDone = tx.processReadTask(async sql => {\n const readable = await sql.unsafe(copyCommand).readable();\n for await (const chunk of readable) {\n chunks.enqueue(chunk as Buffer);\n await acks.dequeue(); // wait for this generator to consume the chunk\n }\n chunks.enqueue('done');\n });\n copyDone.catch(e => chunks.enqueueRejection(e));\n\n let totalBytes = 0;\n let totalMsgs = 0;\n let rowValues: JSONValue[][] = [];\n let bufferedBytes = 0;\n\n const logFlushed = () => {\n lc.debug?.(\n `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n );\n };\n\n // Tracks the row being parsed.\n let row: JSONValue[] = Array.from({length: decoders.length});\n let col = 0;\n\n for (;;) {\n const chunk = await chunks.dequeue();\n if (chunk === 'done') {\n break;\n }\n for (const field of parser.parse(chunk)) {\n row[col] = field === null ? null : decoders[col](field);\n\n if (++col === decoders.length) {\n rowValues.push(row);\n status.rows++;\n row = Array.from({length: decoders.length});\n col = 0;\n }\n }\n bufferedBytes += chunk.byteLength;\n totalBytes += chunk.byteLength;\n\n if (bufferedBytes >= flushThresholdBytes) {\n yield {\n message: {tag: 'backfill', ...backfill, rowValues, status},\n byteSize: bufferedBytes,\n };\n totalMsgs++;\n logFlushed();\n rowValues = [];\n bufferedBytes = 0;\n }\n\n // Signal the read task to pull the next chunk (one-chunk backpressure).\n acks.enqueue();\n }\n\n // Surface any error from the COPY read task (and confirm clean completion).\n await copyDone;\n\n // Flush the last batch of rows.\n if (rowValues.length > 0) {\n yield {\n message: {tag: 'backfill', ...backfill, rowValues, status},\n byteSize: bufferedBytes,\n };\n totalMsgs++;\n logFlushed();\n }\n\n yield {\n message: {tag: 'backfill-completed', ...backfill, status},\n byteSize: 0,\n };\n elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n `(${elapsed} ms)`,\n );\n}\n\n/*\n Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n \n (Note that PG's other LSN-related functions are not scoped to a\n * transaction; this is the only way to get set a transaction at a specific\n * LSN.)\n */\nasync function createSnapshotTransaction(\n lc: LogContext,\n upstreamURI: string,\n db: PostgresDB,\n slotNamePrefix: string,\n) {\n const replicationSession = pgClient(\n lc,\n upstreamURI,\n 'backfill-replication-session',\n {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n },\n );\n const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n try {\n const {snapshot_name: snapshot, consistent_point: lsn} =\n await createReplicationSlot(lc, replicationSession, {\n slotName,\n temporary: true, // deletes the slot when the replicationSession ends\n });\n\n const {init, imported} = importSnapshot(snapshot);\n const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n await imported.dequeue();\n\n const watermark = toStateVersionString(lsn);\n lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n return {tx, watermark};\n } finally {\n await replicationSession.end();\n }\n}\n\nfunction validateSchema(\n tx: TransactionPool,\n publications: string[],\n bf: BackfillRequest,\n watermark: string,\n): Promise<{\n tableSpec: PublishedTableSpec;\n backfill: BackfillParams;\n}> {\n return tx.processReadTask(async sql => {\n const {tables} = await getPublicationInfo(sql, publications);\n const spec = tables.find(\n spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n );\n if (!spec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or dropped`,\n );\n }\n const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n if (spec.schemaOID !== tableMeta.schemaOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Schema no longer corresponds to the original schema`,\n );\n }\n if (spec.oid !== tableMeta.relationOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table no longer corresponds to the original table`,\n );\n }\n if (\n !equals(\n new Set(Object.keys(tableMeta.rowKey)),\n new Set(spec.replicaIdentityColumns),\n )\n ) {\n throw new SchemaIncompatibilityError(\n bf,\n 'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n );\n }\n const allCols = [\n ...Object.entries(tableMeta.rowKey),\n ...Object.entries(bf.columns),\n ];\n for (const [col, val] of allCols) {\n const colSpec = spec.columns[col];\n if (!colSpec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} has been renamed or dropped`,\n );\n }\n const colMeta = v.parse(val, columnMetadataSchema);\n if (colMeta.attNum !== colSpec.pos) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} no longer corresponds to the original column`,\n );\n }\n }\n const backfill: BackfillParams = {\n relation: {\n schema: bf.table.schema,\n name: bf.table.name,\n rowKey: {columns: Object.keys(tableMeta.rowKey)},\n },\n columns: Object.keys(bf.columns).filter(\n col => !(col in tableMeta.rowKey),\n ),\n watermark,\n };\n return {tableSpec: spec, backfill};\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAmEA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GACU;CACjC,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EAAC,sBAAsB,0BAA0B,WAAW,UAChE;CACF,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB;GAGlE,iBAAiB,OAAU;EAM5B,UAAU;CACZ,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,EAAC,WAAW,aAAY,MAAM,eAClC,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EACpD,MAAM,QAAQ,uBAAuB,WAAW,IAAI;EAEpD,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,GACD,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,sBAAsB,WAAW,IAAI,CACvC;GAEA,OAAO,OACL,IACA,IACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,GACD,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EACE,cACA,iBAEF,aACA,QACA,UACA,qBACiC;CAGjC,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CAUA,MAAM,SAAS,IAAI,MAAuB;CAC1C,MAAM,OAAO,IAAI,MAAY;CAE7B,MAAM,WAAW,GAAG,gBAAgB,OAAM,QAAO;EAC/C,MAAM,WAAW,MAAM,IAAI,OAAO,WAAW,EAAE,SAAS;EACxD,WAAW,MAAM,SAAS,UAAU;GAClC,OAAO,QAAQ,KAAe;GAC9B,MAAM,KAAK,QAAQ;EACrB;EACA,OAAO,QAAQ,MAAM;CACvB,CAAC;CACD,SAAS,OAAM,MAAK,OAAO,iBAAiB,CAAC,CAAC;CAE9C,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAGA,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,SAAS;EACP,MAAM,QAAQ,MAAM,OAAO,QAAQ;EACnC,IAAI,UAAU,QACZ;EAEF,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;GAEtD,IAAI,EAAE,QAAQ,SAAS,QAAQ;IAC7B,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,qBAAqB;GACxC,MAAM;IACJ,SAAS;KAAC,KAAK;KAAY,GAAG;KAAU;KAAW;IAAM;IACzD,UAAU;GACZ;GACA;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;EAGA,KAAK,QAAQ;CACf;CAGA,MAAM;CAGN,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GACJ,SAAS;IAAC,KAAK;IAAY,GAAG;IAAU;IAAW;GAAM;GACzD,UAAU;EACZ;EACA;EACA,WAAW;CACb;CAEA,MAAM;EACJ,SAAS;GAAC,KAAK;GAAsB,GAAG;GAAU;EAAM;EACxD,UAAU;CACZ;CACA,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;EACb,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM,SAAS,QAAQ;EAEvB,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,CAC9B;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAYA,OAAO;GAAC,WAAW;GAAM,UAAA;IAVvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAEuB;EAAQ;CACnC,CAAC;AACH"}
1	+ {"version":3,"file":"backfill-stream.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n PG_UNDEFINED_COLUMN,\n PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {Queue} from '../../../../../shared/src/queue.ts';\nimport {randomCharacters} from '../../../../../shared/src/random-values.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {connectPgClient, pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {id} from '../../../types/sql.ts';\nimport {\n SchemaIncompatibilityError,\n type BackfillMessage,\n} from '../common/backfill-manager.ts';\nimport {resumePoint} from '../protocol/backfill-progress.ts';\nimport type {\n BackfillCompleted,\n BackfillProgressMark,\n BackfillRequest,\n DownloadStatus,\n JSONValue,\n} from '../protocol/current.ts';\nimport {\n columnMetadataSchema,\n tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n makeBinarySelectExprs,\n makeDownloadStatements,\n type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicationSlot} from './replication-slots.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag' \| 'progressMarks'>;\n\n/*\n Determines how the progress of a backfill is marked, and from where it is\n * resumed.\n /\ntype Progress = {\n /\n The timeline of the progress marks. For plain tables, this is the table's\n * `relfilenode`, which identifies the physical \"generation\" of the table's\n * data (changed by heap rewrites like `VACUUM FULL`, `CLUSTER`, or\n * `TRUNCATE`), within which `ctid`s are comparable.\n /\n timeline: string;\n\n /\n Whether the progress marks are resumable `ctid`s. Tables without\n * (their own) storage, such as partitioned tables, are not resumable and\n * are instead marked with the number of rows streamed, on a timeline that\n * is unique to the run.\n /\n resumable: boolean;\n\n /* The mark from which the backfill resumes, if any. /\n start: BackfillProgressMark \| undefined;\n};\n\n/\n Encodes a `ctid` text representation, e.g. `(123,4)`, as a\n * fixed-width string that sorts lexicographically in `ctid` order.\n /\nconst CTID_RE = /^\$(\\d+),(\\d+)\$$/;\nconst CTID_MARK_RE = /^(\\d{10})\\.(\\d{5})$/;\n\nexport function ctidToProgressMark(ctid: string): string {\n const match = CTID_RE.exec(ctid);\n assert(match, () => `invalid ctid: ${ctid}`);\n return `${match[1].padStart(10, '0')}.${match[2].padStart(5, '0')}`;\n}\n\n/* The inverse of {@link ctidToProgressMark}, as a `tid` literal. /\nexport function progressMarkToCtid(mark: string): string {\n const match = CTID_MARK_RE.exec(mark);\n assert(match, () => `invalid ctid progress mark: ${mark}`);\n return `(${Number(match[1])},${Number(match[2])})`;\n}\n\nfunction progressMarks(\n previous: BackfillProgressMark \| undefined,\n current: BackfillProgressMark,\n) {\n return previous ? {previous, current} : {current};\n}\n\n/* The `ctid` of each row is appended as the last column of the download. /\nconst CTID_SELECT = 'ctid::text';\n\n/\n Settings that ensure that the COPY scans the table in physical (i.e.\n * `ctid`) order, which is necessary for the progress marks of a resumable\n * backfill to be meaningful. (Monotonicity is nonetheless verified as rows\n * are streamed.)\n \n * Synchronized seqscans can start in the middle of the table.\n * * Parallel scans interleave the rows of different workers.\n * * Index scans follow index order. (A publication row filter could\n * otherwise be satisfied with an index.)\n /\nconst PHYSICAL_ORDER_SETTINGS = [\n 'synchronize_seqscans = off',\n 'max_parallel_workers_per_gather = 0',\n 'enable_indexscan = off',\n 'enable_indexonlyscan = off',\n 'enable_bitmapscan = off',\n];\n\nfunction rowCountMark(rows: number): string {\n return String(rows).padStart(16, '0');\n}\n\ntype StreamOptions = {\n /\n The number of bytes at which to flush a batch of rows in a\n * backfill message. Defaults to Node's getDefaultHighWatermark().\n /\n flushThresholdBytes?: number \| undefined;\n\n /\n Use text-format COPY instead of binary COPY.\n * Binary is faster and handles all types (unknown types are cast to\n * `::text` in the SELECT). This flag exists as an escape hatch to\n * revert to the old code path if needed.\n /\n textCopy?: boolean \| undefined;\n\n /\n @visibleForTesting\n * Called after the snapshot transaction is opened, before the table is\n * locked, e.g. to run a concurrent heap rewrite in between.\n /\n afterSnapshotForTesting?: (() => Promise<void>) \| undefined;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 1024;\n\n// Matches the exact clauses emitted by makeDownloadStatements; quoted\n// identifiers like \"limit\" won't match because they lack the surrounding\n// whitespace.\nconst SAMPLE_OR_LIMIT_RE = /\\sTABLESAMPLE\\s+BERNOULLI\\b\|\\sLIMIT\\s+\\d/i;\n\n/*\n Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n /\nexport async function streamBackfill(\n lc: LogContext,\n upstreamURI: string,\n {slot, publications}: Pick<Replica, 'slot' \| 'publications'>,\n bf: BackfillRequest,\n opts: StreamOptions = {},\n): AsyncGenerator<BackfillMessage> {\n lc = lc\n .withContext('component', 'backfill')\n .withContext('table', bf.table.name);\n\n const {\n flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE,\n textCopy = false,\n afterSnapshotForTesting,\n } = opts;\n const db = await connectPgClient(lc, upstreamURI, 'backfill-stream', {\n // The COPY is a single stream that must outlive the entire table download,\n // so allow a very long (24h) connection lifetime.\n ['max_lifetime']: 24 * 60 * 60,\n // A backfill COPY is a background process that can be preempted by\n // upstream replication changes, potentially requiring it to sit idle for\n // long stretches. Use TCP keepalive to detect dead connections instead\n // of the inactivity watchdog, which could kill connnections that are\n // taking a back seat for upstream replication changes.\n liveness: 'keepalive',\n });\n let tx: TransactionPool \| undefined;\n let watermark: string;\n try {\n ({tx, watermark} = await createSnapshotTransaction(\n lc,\n upstreamURI,\n db,\n slot,\n ));\n await afterSnapshotForTesting?.();\n const {tableSpec, backfill, progress} = await validateSchema(\n lc,\n tx,\n publications,\n bf,\n watermark,\n );\n\n // Note: validateSchema ensures that the rowKey and columns are disjoint\n const {relation, columns} = backfill;\n const cols = [...relation.rowKey.columns, ...columns];\n // Resumable backfills are streamed in physical (i.e. ctid) order, which\n // is the natural order of a sequential or TID range scan (as enforced by\n // the settings in `stream()`). An ORDER BY is not used, as Postgres does\n // not consider either scan to be ordered, and would unnecessarily sort\n // the table.\n const order =\n progress.resumable && progress.start\n ? {\n after: /sql/ `ctid > '${progressMarkToCtid(progress.start.progressMark)}'::tid`,\n }\n : undefined;\n const stmts = makeDownloadStatements(\n tableSpec,\n cols,\n undefined,\n undefined,\n [...cols.map(col => id(col)), CTID_SELECT],\n order,\n );\n if (progress.start) {\n lc.info?.(\n `resuming backfill from ${progress.start.progressMark} ` +\n `(timeline ${progress.timeline})`,\n );\n }\n\n if (textCopy) {\n const types = await getTypeParsers(db, {returnJsonAsString: true});\n yield* stream(\n lc,\n tx,\n backfill,\n progress,\n stmts,\n `COPY (${stmts.select}) TO STDOUT`,\n new TsvParser(),\n cols.map(col => {\n const parser = types.getTypeParser(tableSpec.columns[col].typeOID);\n return (text: string) => parser(text) as JSONValue;\n }),\n (text: string) => text,\n flushThresholdBytes,\n );\n } else {\n const binaryStmts = makeDownloadStatements(\n tableSpec,\n cols,\n undefined,\n undefined,\n [...makeBinarySelectExprs(tableSpec, cols), CTID_SELECT],\n order,\n );\n\n yield* stream(\n lc,\n tx,\n backfill,\n progress,\n stmts,\n `COPY (${binaryStmts.select}) TO STDOUT WITH (FORMAT binary)`,\n new BinaryCopyParser(),\n cols.map(col => {\n const spec = tableSpec.columns[col];\n const decoder = hasBinaryDecoder(spec)\n ? makeBinaryDecoder(spec)\n : textCastDecoder;\n return (buf: Buffer) => decoder(buf) as unknown as JSONValue;\n }),\n (buf: Buffer) => buf.toString('utf8'), // ctid::text\n flushThresholdBytes,\n );\n }\n } catch (e) {\n // Although we make the best effort to validate the schema at the\n // transaction snapshot, certain forms of `ALTER TABLE` are not\n // MVCC safe and not \"frozen\" in the snapshot:\n //\n // https://www.postgresql.org/docs/current/mvcc-caveats.html\n //\n // Handle these errors as schema incompatibility errors rather than\n // unknown runtime errors.\n if (\n e instanceof postgres.PostgresError &&\n (e.code === PG_UNDEFINED_TABLE \|\| e.code === PG_UNDEFINED_COLUMN)\n ) {\n throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n }\n throw e;\n } finally {\n tx?.setDone();\n // Workaround postgres.js hanging at the end of some COPY commands:\n // https://github.com/porsager/postgres/issues/499\n void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n }\n}\n\nasync function* stream<T>(\n lc: LogContext,\n tx: TransactionPool,\n backfill: BackfillParams,\n {timeline, resumable, start: startMark}: Progress,\n {\n getTotalRows,\n getTotalBytes,\n }: Pick<DownloadStatements, 'getTotalRows' \| 'getTotalBytes'>,\n copyCommand: string,\n parser: {parse(chunk: Buffer): Iterable<T \| null>},\n decoders: ((field: T) => JSONValue)[],\n ctidDecoder: (field: T) => string,\n flushThresholdBytes: number,\n): AsyncGenerator<BackfillMessage> {\n // Backfill must read every row: TABLESAMPLE / LIMIT are reserved for shadow\n // sync and must never appear in a backfill COPY.\n assert(\n !SAMPLE_OR_LIMIT_RE.test(copyCommand),\n `backfill COPY must not sample or limit: ${copyCommand}`,\n );\n const start = performance.now();\n const [rows, bytes] = await tx.processReadTask(sql =>\n Promise.all([\n sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n ]),\n );\n const status: DownloadStatus = {\n rows: 0,\n totalRows: Number(rows[0].totalRows),\n totalBytes: Number(bytes[0].totalBytes),\n };\n\n let elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Computed total rows and bytes for: ${copyCommand} (${elapsed} ms)`,\n {\n status,\n },\n );\n\n // Drain the COPY stream from within a single read task so that the\n // TransactionPool worker holds the transaction for the entire duration of\n // the COPY, rather than incorrectly considering the worker \"idle\" and\n // attempting to send keepalives on it.\n //\n // Chunks are bridged out to this generator via a Queue, with a corresponding\n // acks Queue providing strict one-chunk-at-a-time backpressure. This is\n // necessary because it is not possible to \"yield\" from within the read task.\n const chunks = new Queue<Buffer \| 'done'>();\n const acks = new Queue<void>();\n\n const copyDone = tx.processReadTask(async sql => {\n // SET LOCAL applies to the (snapshot) transaction of this read task.\n for (const setting of PHYSICAL_ORDER_SETTINGS) {\n await sql.unsafe(`SET LOCAL ${setting}`);\n }\n const readable = await sql.unsafe(copyCommand).readable();\n for await (const chunk of readable) {\n chunks.enqueue(chunk as Buffer);\n await acks.dequeue(); // wait for this generator to consume the chunk\n }\n chunks.enqueue('done');\n });\n copyDone.catch(e => chunks.enqueueRejection(e));\n\n let totalBytes = 0;\n let totalMsgs = 0;\n let rowValues: JSONValue[][] = [];\n let bufferedBytes = 0;\n\n const logFlushed = () => {\n lc.debug?.(\n `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n );\n };\n\n // The mark of the last row that was streamed, and that of the last row of\n // the previous message.\n let current: BackfillProgressMark \| undefined = startMark;\n let previous: BackfillProgressMark \| undefined = startMark;\n const markOf = (progressMark: string) => ({progressMark, timeline});\n\n // The number of columns of each row, followed by the ctid.\n const numCols = decoders.length + 1;\n\n // Tracks the row being parsed.\n let row: JSONValue[] = Array.from({length: decoders.length});\n let col = 0;\n\n for (;;) {\n const chunk = await chunks.dequeue();\n if (chunk === 'done') {\n break;\n }\n for (const field of parser.parse(chunk)) {\n if (col < decoders.length) {\n row[col] = field === null ? null : decoders[col](field);\n } else {\n const mark = resumable\n ? ctidToProgressMark(ctidDecoder(field as T))\n : rowCountMark(status.rows + 1);\n if (current !== undefined && mark <= current.progressMark) {\n // This should not happen given the PHYSICAL_ORDER_SETTINGS, but\n // the correctness of the progress marks depends on it.\n throw new Error(\n `backfill rows are not in ctid order ` +\n `(${mark} after ${current.progressMark})`,\n );\n }\n current = markOf(mark);\n }\n\n if (++col === numCols) {\n rowValues.push(row);\n status.rows++;\n row = Array.from({length: decoders.length});\n col = 0;\n }\n }\n bufferedBytes += chunk.byteLength;\n totalBytes += chunk.byteLength;\n\n if (bufferedBytes >= flushThresholdBytes && rowValues.length > 0) {\n yield {\n message: {\n tag: 'backfill',\n ...backfill,\n rowValues,\n status,\n progressMarks: progressMarks(previous, must(current)),\n },\n byteSize: bufferedBytes,\n };\n previous = current;\n totalMsgs++;\n logFlushed();\n rowValues = [];\n bufferedBytes = 0;\n }\n\n // Signal the read task to pull the next chunk (one-chunk backpressure).\n acks.enqueue();\n }\n\n // Surface any error from the COPY read task (and confirm clean completion).\n await copyDone;\n\n // Flush the last batch of rows.\n if (rowValues.length > 0) {\n yield {\n message: {\n tag: 'backfill',\n ...backfill,\n rowValues,\n status,\n progressMarks: progressMarks(previous, must(current)),\n },\n byteSize: bufferedBytes,\n };\n previous = current;\n totalMsgs++;\n logFlushed();\n }\n\n yield {\n message: {\n tag: 'backfill-completed',\n ...backfill,\n status,\n // `previous` is only unset for a backfill from scratch with no rows.\n progressMarks: previous ? {previous} : {},\n },\n byteSize: 0,\n };\n elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n `(${elapsed} ms)`,\n );\n}\n\n/*\n Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n \n (Note that PG's other LSN-related functions are not scoped to a\n * transaction; this is the only way to get set a transaction at a specific\n * LSN.)\n /\nasync function createSnapshotTransaction(\n lc: LogContext,\n upstreamURI: string,\n db: PostgresDB,\n slotNamePrefix: string,\n) {\n const replicationSession = pgClient(\n lc,\n upstreamURI,\n 'backfill-replication-session',\n {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n },\n );\n const slotName = `${slotNamePrefix}_bf_${Date.now()}`;\n try {\n const {snapshot_name: snapshot, consistent_point: lsn} =\n await createReplicationSlot(lc, replicationSession, {\n slotName,\n temporary: true, // deletes the slot when the replicationSession ends\n });\n\n const {init, imported} = importSnapshot(snapshot);\n const tx = new TransactionPool(lc, {mode: READONLY, init}).run(db);\n await imported.dequeue();\n\n const watermark = toStateVersionString(lsn);\n lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n return {tx, watermark};\n } finally {\n await replicationSession.end();\n }\n}\n\nfunction validateSchema(\n lc: LogContext,\n tx: TransactionPool,\n publications: string[],\n bf: BackfillRequest,\n watermark: string,\n): Promise<{\n tableSpec: PublishedTableSpec;\n backfill: BackfillParams;\n progress: Progress;\n}> {\n return tx.processReadTask(async sql => {\n const {tables} = await getPublicationInfo(sql, publications);\n const spec = tables.find(\n spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n );\n if (!spec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or dropped`,\n );\n }\n const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n if (spec.schemaOID !== tableMeta.schemaOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Schema no longer corresponds to the original schema`,\n );\n }\n if (spec.oid !== tableMeta.relationOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table no longer corresponds to the original table`,\n );\n }\n if (\n !equals(\n new Set(Object.keys(tableMeta.rowKey)),\n new Set(spec.replicaIdentityColumns),\n )\n ) {\n throw new SchemaIncompatibilityError(\n bf,\n 'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n );\n }\n const allCols = [\n ...Object.entries(tableMeta.rowKey),\n ...Object.entries(bf.columns).map(([col, {id}]) => [col, id] as const),\n ];\n for (const [col, val] of allCols) {\n const colSpec = spec.columns[col];\n if (!colSpec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} has been renamed or dropped`,\n );\n }\n const colMeta = v.parse(val, columnMetadataSchema);\n if (colMeta.attNum !== colSpec.pos) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} no longer corresponds to the original column`,\n );\n }\n }\n const backfill: BackfillParams = {\n relation: {\n schema: bf.table.schema,\n name: bf.table.name,\n rowKey: {columns: Object.keys(tableMeta.rowKey)},\n },\n columns: Object.keys(bf.columns).filter(\n col => !(col in tableMeta.rowKey),\n ),\n watermark,\n };\n const progress = await getProgress(lc, sql, spec.oid, bf);\n return {tableSpec: spec, backfill, progress};\n });\n}\n\n/\n Determines the timeline of the table's progress marks (as of the snapshot),\n * and whether the backfill can be resumed from the requested progress.\n \n The table is locked (in ACCESS SHARE mode, as the COPY would, for the rest\n * of the transaction) so that its storage cannot be rewritten (by `VACUUM\n * FULL`, `CLUSTER`, `TRUNCATE`, or a rewriting `ALTER TABLE`, all of which\n * require ACCESS EXCLUSIVE) between reading the timeline and the COPY. Note\n * that `pg_relation_filenode()` reflects the table's current storage (only\n * refreshed upon acquiring a lock), whereas `pg_class.relfilenode` is read at\n * the transaction snapshot. If they differ, the storage was rewritten after the\n * snapshot, and the COPY would scan the new storage with the old snapshot,\n * which is not MVCC-safe for all rewrites (e.g. the table appears empty after\n * a rewriting `ALTER TABLE`). The backfill is then retried at a new snapshot.\n /\nasync function getProgress(\n lc: LogContext,\n sql: postgres.Sql,\n relationOID: number,\n bf: BackfillRequest,\n): Promise<Progress> {\n const table = `${id(bf.table.schema)}.${id(bf.table.name)}`;\n await sql.unsafe(`LOCK TABLE ${table} IN ACCESS SHARE MODE`);\n const [{lockedOID, relkind, snapshotFilenode, filenode}] = await sql<\n {\n lockedOID: number \| null;\n relkind: string;\n snapshotFilenode: string;\n filenode: string \| null;\n }[]\n >`\n SELECT to_regclass(${table})::oid::int8 AS \"lockedOID\",\n relkind,\n relfilenode::text AS \"snapshotFilenode\",\n pg_relation_filenode(oid)::text AS filenode\n FROM pg_class WHERE oid = ${relationOID}`;\n if (Number(lockedOID) !== relationOID) {\n // The (current) table with the name is not the one at the snapshot.\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or replaced`,\n );\n }\n if (relkind !== 'r' \|\| filenode === null) {\n // Not a plain table (e.g. a partitioned table), for which ctids are not\n // unique. Such backfills are not resumable, which is achieved by using a\n // timeline that is unique to the run.\n return {\n timeline: `${relkind}:${randomCharacters(12)}`,\n resumable: false,\n start: undefined,\n };\n }\n if (filenode !== snapshotFilenode) {\n throw new Error(\n `${bf.table.schema}.${bf.table.name} was rewritten after the backfill ` +\n `snapshot (relfilenode ${snapshotFilenode} => ${filenode})`,\n );\n }\n const timeline = filenode;\n const requested = resumePoint(bf);\n if (requested && requested.timeline !== timeline) {\n lc.info?.(\n `restarting backfill from scratch: the requested progress is on ` +\n `timeline ${requested.timeline} but the table is on ${timeline}`,\n );\n return {timeline, resumable: true, start: undefined};\n }\n if (requested && !(await canResume(sql, relationOID, bf))) {\n lc.info?.(\n `restarting backfill from scratch: the table has TOASTed values ` +\n `and backfilling columns [${Object.keys(bf.columns).join(',')}] ` +\n `are TOAST-able`,\n );\n return {timeline, resumable: true, start: undefined};\n }\n return {timeline, resumable: true, start: requested};\n}\n\n/\n Whether resuming a backfill (in a new snapshot) is guaranteed to deliver\n * the values of every row that the preceding snapshot(s) had not reached.\n \n Within a single snapshot, every row is scanned exactly once, since the\n * version visible to the snapshot stays in place while it is held. Across\n * snapshots, however, a row that had not yet been reached (i.e. beyond the\n * resume mark `M`) can be updated between the snapshots, with its new\n * version placed at or before `M` (a non-HOT update into free space in an\n * earlier page, or a HOT update into a lower line pointer of `M`'s page).\n * Neither snapshot then scans the row, and it is only delivered by the\n * replication stream. The stream's UPDATE contains the full row, _except_\n * for unchanged values stored out-of-line in TOAST, which pgoutput omits\n * (unless the table has REPLICA IDENTITY FULL). A backfilling column with\n * such a value would be published without it.\n \n Resuming is therefore only done if no backfilling column can be TOASTed\n * (i.e. all have `attstorage = 'p'`, as for fixed-width types), or if the\n * table has no TOAST data. The latter is safe because an omitted value must\n * still be in the TOAST relation at the new snapshot (unless its row has since\n * been deleted, which the replication stream does deliver), and conservative\n * because the TOAST relation does not shrink to empty without a heap rewrite,\n * which changes the timeline.\n \n Potential optimization for TOAST-able backfills: have the resumed run also\n * deliver the rows at or before `M` that were modified since the snapshot of\n * the run that started the backfill from scratch. That run's snapshot `xmin`\n * (`T_origin`, as an xid8) would be encoded in the timeline, i.e.\n * `timeline = \"<relfilenode>:<T_origin>\"`, and carried over (rather than\n * replaced) by resumed runs, so that:\n * * Within a chain of runs, every row version with `xmin < T_origin` stayed\n * in place across all of the runs' snapshots and was therefore scanned\n * exactly once, and all other rows are re-delivered by the resumed run\n * (`ctid > M OR xmin >= T_origin`).\n * * Continuations from a different chain (i.e. with a newer `T_origin`) are\n * not accepted by subscribers, and requests from different chains are\n * merged into a backfill from scratch, as the timelines differ. Note that\n * using the _latest_ `T` would be incorrect: rows that moved before `M`\n * between the earlier snapshots would never be re-delivered.\n * The resumed run would scan the whole table (but only deliver the modified\n * rows), comparing the 32-bit `xmin` with `T_origin` modulo 2^32, which\n * requires falling back to a backfill from scratch once the current xid is\n * 2^31 or more past `T_origin` (i.e. wraparound). This also relies on the\n * `xmin` system column returning the raw xmin of frozen tuples, which should\n * be verified.\n */\nasync function canResume(\n sql: postgres.Sql,\n relationOID: number,\n bf: BackfillRequest,\n): Promise<boolean> {\n const [{toastable, toasted}] = await sql<\n {toastable: boolean; toasted: boolean}[]\n >`\n SELECT\n EXISTS (\n SELECT 1 FROM pg_attribute\n WHERE attrelid = ${relationOID}\n AND attname IN ${sql(Object.keys(bf.columns))}\n AND attstorage <> 'p'\n ) AS toastable,\n reltoastrelid <> 0 AND pg_relation_size(reltoastrelid) > 0 AS toasted\n FROM pg_class WHERE oid = ${relationOID}`;\n return !toastable \|\| !toasted;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAkFA,IAAM,UAAU;AAChB,IAAM,eAAe;AAErB,SAAgB,mBAAmB,MAAsB;CACvD,MAAM,QAAQ,QAAQ,KAAK,IAAI;CAC/B,OAAO,aAAa,iBAAiB,MAAM;CAC3C,OAAO,GAAG,MAAM,GAAG,SAAS,IAAI,GAAG,EAAE,GAAG,MAAM,GAAG,SAAS,GAAG,GAAG;AAClE;;AAGA,SAAgB,mBAAmB,MAAsB;CACvD,MAAM,QAAQ,aAAa,KAAK,IAAI;CACpC,OAAO,aAAa,+BAA+B,MAAM;CACzD,OAAO,IAAI,OAAO,MAAM,EAAE,EAAE,GAAG,OAAO,MAAM,EAAE,EAAE;AAClD;AAEA,SAAS,cACP,UACA,SACA;CACA,OAAO,WAAW;EAAC;EAAU;CAAO,IAAI,EAAC,QAAO;AAClD;;AAGA,IAAM,cAAc;;;;;;;;;;;;AAapB,IAAM,0BAA0B;CAC9B;CACA;CACA;CACA;CACA;AACF;AAEA,SAAS,aAAa,MAAsB;CAC1C,OAAO,OAAO,IAAI,EAAE,SAAS,IAAI,GAAG;AACtC;AA4BA,IAAM,2BAA2B,KAAK;AAKtC,IAAM,qBAAqB;;;;;;;AAQ3B,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,CAAC,GACU;CACjC,KAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;CAErC,MAAM,EACJ,sBAAsB,0BACtB,WAAW,OACX,4BACE;CACJ,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,mBAAmB;GAGlE,iBAAiB,OAAU;EAM5B,UAAU;CACZ,CAAC;CACD,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,CAAC,CAAC,IAAI,aAAa,MAAM,0BACvB,IACA,aACA,IACA,IACF;EACA,MAAM,0BAA0B;EAChC,MAAM,EAAC,WAAW,UAAU,aAAY,MAAM,eAC5C,IACA,IACA,cACA,IACA,SACF;EAGA,MAAM,EAAC,UAAU,YAAW;EAC5B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;EAMpD,MAAM,QACJ,SAAS,aAAa,SAAS,QAC3B,EACE,OAAe,WAAW,mBAAmB,SAAS,MAAM,YAAY,EAAE,QAC5E,IACA,KAAA;EACN,MAAM,QAAQ,uBACZ,WACA,MACA,KAAA,GACA,KAAA,GACA,CAAC,GAAG,KAAK,KAAI,QAAO,GAAG,GAAG,CAAC,GAAG,WAAW,GACzC,KACF;EACA,IAAI,SAAS,OACX,GAAG,OACD,0BAA0B,SAAS,MAAM,aAAa,aACvC,SAAS,SAAS,EACnC;EAGF,IAAI,UAAU;GACZ,MAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;GACjE,OAAO,OACL,IACA,IACA,UACA,UACA,OACA,SAAS,MAAM,OAAO,cACtB,IAAI,UAAU,GACd,KAAK,KAAI,QAAO;IACd,MAAM,SAAS,MAAM,cAAc,UAAU,QAAQ,KAAK,OAAO;IACjE,QAAQ,SAAiB,OAAO,IAAI;GACtC,CAAC,IACA,SAAiB,MAClB,mBACF;EACF,OAAO;GACL,MAAM,cAAc,uBAClB,WACA,MACA,KAAA,GACA,KAAA,GACA,CAAC,GAAG,sBAAsB,WAAW,IAAI,GAAG,WAAW,GACvD,KACF;GAEA,OAAO,OACL,IACA,IACA,UACA,UACA,OACA,SAAS,YAAY,OAAO,mCAC5B,IAAI,iBAAiB,GACrB,KAAK,KAAI,QAAO;IACd,MAAM,OAAO,UAAU,QAAQ;IAC/B,MAAM,UAAU,iBAAiB,IAAI,IACjC,kBAAkB,IAAI,IACtB;IACJ,QAAQ,QAAgB,QAAQ,GAAG;GACrC,CAAC,IACA,QAAgB,IAAI,SAAS,MAAM,GACpC,mBACF;EACF;CACF,SAAS,GAAG;EASV,IACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAE7C,MAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,EAAC,CAAC;EAEhE,MAAM;CACR,UAAU;EACR,IAAI,QAAQ;EAGZ,GAAQ,IAAI,EAAE,OAAM,MAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;CAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EAAC,UAAU,WAAW,OAAO,aAC7B,EACE,cACA,iBAEF,aACA,QACA,UACA,aACA,qBACiC;CAGjC,OACE,CAAC,mBAAmB,KAAK,WAAW,GACpC,2CAA2C,aAC7C;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,CAAC,MAAM,SAAS,MAAM,GAAG,iBAAgB,QAC7C,QAAQ,IAAI,CACV,IAAI,OAA8B,YAAY,GAC9C,IAAI,OAA+B,aAAa,CAClD,CAAC,CACH;CACA,MAAM,SAAyB;EAC7B,MAAM;EACN,WAAW,OAAO,KAAK,GAAG,SAAS;EACnC,YAAY,OAAO,MAAM,GAAG,UAAU;CACxC;CAEA,IAAI,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CACnD,GAAG,OACD,sCAAsC,YAAY,IAAI,QAAQ,OAC9D,EACE,OACF,CACF;CAUA,MAAM,SAAS,IAAI,MAAuB;CAC1C,MAAM,OAAO,IAAI,MAAY;CAE7B,MAAM,WAAW,GAAG,gBAAgB,OAAM,QAAO;EAE/C,KAAK,MAAM,WAAW,yBACpB,MAAM,IAAI,OAAO,aAAa,SAAS;EAEzC,MAAM,WAAW,MAAM,IAAI,OAAO,WAAW,EAAE,SAAS;EACxD,WAAW,MAAM,SAAS,UAAU;GAClC,OAAO,QAAQ,KAAe;GAC9B,MAAM,KAAK,QAAQ;EACrB;EACA,OAAO,QAAQ,MAAM;CACvB,CAAC;CACD,SAAS,OAAM,MAAK,OAAO,iBAAiB,CAAC,CAAC;CAE9C,IAAI,aAAa;CACjB,IAAI,YAAY;CAChB,IAAI,YAA2B,CAAC;CAChC,IAAI,gBAAgB;CAEpB,MAAM,mBAAmB;EACvB,GAAG,QACD,WAAW,UAAU,OAAO,SAAS,cAAc,sBACjC,OAAO,KAAK,SAAS,UAAU,UAAU,WAAW,EACxE;CACF;CAIA,IAAI,UAA4C;CAChD,IAAI,WAA6C;CACjD,MAAM,UAAU,kBAA0B;EAAC;EAAc;CAAQ;CAGjE,MAAM,UAAU,SAAS,SAAS;CAGlC,IAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;CAC3D,IAAI,MAAM;CAEV,SAAS;EACP,MAAM,QAAQ,MAAM,OAAO,QAAQ;EACnC,IAAI,UAAU,QACZ;EAEF,KAAK,MAAM,SAAS,OAAO,MAAM,KAAK,GAAG;GACvC,IAAI,MAAM,SAAS,QACjB,IAAI,OAAO,UAAU,OAAO,OAAO,SAAS,KAAK,KAAK;QACjD;IACL,MAAM,OAAO,YACT,mBAAmB,YAAY,KAAU,CAAC,IAC1C,aAAa,OAAO,OAAO,CAAC;IAChC,IAAI,YAAY,KAAA,KAAa,QAAQ,QAAQ,cAG3C,MAAM,IAAI,MACR,wCACM,KAAK,SAAS,QAAQ,aAAa,EAC3C;IAEF,UAAU,OAAO,IAAI;GACvB;GAEA,IAAI,EAAE,QAAQ,SAAS;IACrB,UAAU,KAAK,GAAG;IAClB,OAAO;IACP,MAAM,MAAM,KAAK,EAAC,QAAQ,SAAS,OAAM,CAAC;IAC1C,MAAM;GACR;EACF;EACA,iBAAiB,MAAM;EACvB,cAAc,MAAM;EAEpB,IAAI,iBAAiB,uBAAuB,UAAU,SAAS,GAAG;GAChE,MAAM;IACJ,SAAS;KACP,KAAK;KACL,GAAG;KACH;KACA;KACA,eAAe,cAAc,UAAU,KAAK,OAAO,CAAC;IACtD;IACA,UAAU;GACZ;GACA,WAAW;GACX;GACA,WAAW;GACX,YAAY,CAAC;GACb,gBAAgB;EAClB;EAGA,KAAK,QAAQ;CACf;CAGA,MAAM;CAGN,IAAI,UAAU,SAAS,GAAG;EACxB,MAAM;GACJ,SAAS;IACP,KAAK;IACL,GAAG;IACH;IACA;IACA,eAAe,cAAc,UAAU,KAAK,OAAO,CAAC;GACtD;GACA,UAAU;EACZ;EACA,WAAW;EACX;EACA,WAAW;CACb;CAEA,MAAM;EACJ,SAAS;GACP,KAAK;GACL,GAAG;GACH;GAEA,eAAe,WAAW,EAAC,SAAQ,IAAI,CAAC;EAC1C;EACA,UAAU;CACZ;CACA,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;CAC/C,GAAG,OACD,sBAAsB,OAAO,KAAK,SAAS,UAAU,SAAS,WAAW,UACnE,QAAQ,KAChB;AACF;;;;;;;;;;AAWA,eAAe,0BACb,IACA,aACA,IACA,gBACA;CACA,MAAM,qBAAqB,SACzB,IACA,aACA,gCACA;GACG,gBAAgB;EACjB,YAAY,EAAC,aAAa,WAAU;CACtC,CACF;CACA,MAAM,WAAW,GAAG,eAAe,MAAM,KAAK,IAAI;CAClD,IAAI;EACF,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB;GAClD;GACA,WAAW;EACb,CAAC;EAEH,MAAM,EAAC,MAAM,aAAY,eAAe,QAAQ;EAChD,MAAM,KAAK,IAAI,gBAAgB,IAAI;GAAC,MAAM;GAAU;EAAI,CAAC,EAAE,IAAI,EAAE;EACjE,MAAM,SAAS,QAAQ;EAEvB,MAAM,YAAY,qBAAqB,GAAG;EAC1C,GAAG,OAAO,sCAAsC,IAAI,IAAI,UAAU,EAAE;EACpE,OAAO;GAAC;GAAI;EAAS;CACvB,UAAU;EACR,MAAM,mBAAmB,IAAI;CAC/B;AACF;AAEA,SAAS,eACP,IACA,IACA,cACA,IACA,WAKC;CACD,OAAO,GAAG,gBAAgB,OAAM,QAAO;EACrC,MAAM,EAAC,WAAU,MAAM,mBAAmB,KAAK,YAAY;EAC3D,MAAM,OAAO,OAAO,MAClB,SAAQ,KAAK,WAAW,GAAG,MAAM,UAAU,KAAK,SAAS,GAAG,MAAM,IACpE;EACA,IAAI,CAAC,MACH,MAAM,IAAI,2BACR,IACA,mCACF;EAEF,MAAM,YAAY,MAAQ,GAAG,MAAM,UAAU,mBAAmB;EAChE,IAAI,KAAK,cAAc,UAAU,WAC/B,MAAM,IAAI,2BACR,IACA,qDACF;EAEF,IAAI,KAAK,QAAQ,UAAU,aACzB,MAAM,IAAI,2BACR,IACA,mDACF;EAEF,IACE,CAAC,OACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC,GACrC,IAAI,IAAI,KAAK,sBAAsB,CACrC,GAEA,MAAM,IAAI,2BACR,IACA,iDACF;EAEF,MAAM,UAAU,CACd,GAAG,OAAO,QAAQ,UAAU,MAAM,GAClC,GAAG,OAAO,QAAQ,GAAG,OAAO,EAAE,KAAK,CAAC,KAAK,EAAC,UAAS,CAAC,KAAK,EAAE,CAAU,CACvE;EACA,KAAK,MAAM,CAAC,KAAK,QAAQ,SAAS;GAChC,MAAM,UAAU,KAAK,QAAQ;GAC7B,IAAI,CAAC,SACH,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,6BAChB;GAGF,IADgB,MAAQ,KAAK,oBACzB,EAAQ,WAAW,QAAQ,KAC7B,MAAM,IAAI,2BACR,IACA,UAAU,IAAI,8CAChB;EAEJ;EAaA,OAAO;GAAC,WAAW;GAAM,UAAA;IAXvB,UAAU;KACR,QAAQ,GAAG,MAAM;KACjB,MAAM,GAAG,MAAM;KACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAC;IACjD;IACA,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE,QAC/B,QAAO,EAAE,OAAO,UAAU,OAC5B;IACA;GAGuB;GAAU,UAAA,MADZ,YAAY,IAAI,KAAK,KAAK,KAAK,EAAE;EACb;CAC7C,CAAC;AACH;;;;;;;;;;;;;;;;AAiBA,eAAe,YACb,IACA,KACA,aACA,IACmB;CACnB,MAAM,QAAQ,GAAG,GAAG,GAAG,MAAM,MAAM,EAAE,GAAG,GAAG,GAAG,MAAM,IAAI;CACxD,MAAM,IAAI,OAAO,cAAc,MAAM,sBAAsB;CAC3D,MAAM,CAAC,EAAC,WAAW,SAAS,kBAAkB,cAAa,MAAM,GAOhE;yBACsB,MAAM;;;;kCAIG;CAChC,IAAI,OAAO,SAAS,MAAM,aAExB,MAAM,IAAI,2BACR,IACA,oCACF;CAEF,IAAI,YAAY,OAAO,aAAa,MAIlC,OAAO;EACL,UAAU,GAAG,QAAQ,GAAG,iBAAiB,EAAE;EAC3C,WAAW;EACX,OAAO,KAAA;CACT;CAEF,IAAI,aAAa,kBACf,MAAM,IAAI,MACR,GAAG,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAK,0DACT,iBAAiB,MAAM,SAAS,EAC7D;CAEF,MAAM,WAAW;CACjB,MAAM,YAAY,YAAY,EAAE;CAChC,IAAI,aAAa,UAAU,aAAa,UAAU;EAChD,GAAG,OACD,2EACc,UAAU,SAAS,uBAAuB,UAC1D;EACA,OAAO;GAAC;GAAU,WAAW;GAAM,OAAO,KAAA;EAAS;CACrD;CACA,IAAI,aAAa,CAAE,MAAM,UAAU,KAAK,aAAa,EAAE,GAAI;EACzD,GAAG,OACD,2FAC8B,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,iBAElE;EACA,OAAO;GAAC;GAAU,WAAW;GAAM,OAAO,KAAA;EAAS;CACrD;CACA,OAAO;EAAC;EAAU,WAAW;EAAM,OAAO;CAAS;AACrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgDA,eAAe,UACb,KACA,aACA,IACkB;CAClB,MAAM,CAAC,EAAC,WAAW,aAAY,MAAM,GAEpC;;;;6BAI0B,YAAY;6BACZ,IAAI,OAAO,KAAK,GAAG,OAAO,CAAC,EAAE;;;;kCAIxB;CAChC,OAAO,CAAC,aAAa,CAAC;AACxB"}

package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts CHANGED Viewed

@@ -88,6 +88,11 @@ export declare function verifyShadowReplica(lc: LogContext, db: Database, publis
     tables: PublishedTableSpec[];
     indexes: IndexSpec[];
 }, rowsByTable: ReadonlyMap<string, number>): void;
+/**
+ * The row filter of the table's publications, as a parenthesized boolean
+ * expression, or `null` if the table is published without a filter.
+ */
+export declare function publicationRowFilter(table: PublishedTableSpec): string | null;
 export declare const INSERT_BATCH_SIZE = 50;
 export type DownloadStatements = {
     select: string;
@@ -104,8 +109,12 @@ export declare function makeBinarySelectExprs(table: PublishedTableSpec, cols: s
  * that order. Used by resumable backfills; see `backfill-resume.ts`.
  */
 export type DownloadOrder = {
-    /** The `ORDER BY` expression, e.g. from `orderByRowKey()`. */
-    readonly by: string;
+    /**
+     * The `ORDER BY` expression, e.g. from `orderByRowKey()`. If unset, the
+     * natural order of the scan is used (e.g. physical order for `ctid`-based
+     * resumption).
+     */
+    readonly by?: string | undefined;
     /**
      * A boolean SQL expression restricting the download to rows after a mark,
      * e.g. from `resumeWhere()`. It is ANDed with the publication row filter,

package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGzE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAoBzD,OAAO,KAAK,EAAC,SAAS,EAAE,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAaxE,OAAO,EAEL,KAAK,UAAU,EAGhB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAG1D,OAAO,EAEL,0BAA0B,EAC3B,MAAM,wCAAwC,CAAC;~~AAShD~~,OAAO,EAML,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC/B,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9C,2BAA2B,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClD;;;;;;OAMG;IACH,MAAM,CAAC,EACH;QACE,6DAA6D;QAC7D,UAAU,EAAE,MAAM,CAAC;QACnB;;;WAGG;QACH,eAAe,EAAE,MAAM,CAAC;KACzB,GACD,SAAS,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,EAAC,KAAK,EAAE,QAAQ,EAAC,GAAE,cAA2C,GAC7D,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CA+RnC;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,aAAa,EACtB,WAAW,CAAC,EAAE,IAAI,CAAC,kBAAkB,EAAE,UAAU,CAAC,GACjD,OAAO,CAAC,IAAI,CAAC,CA+Bf;AAgKD;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,OAAO,EAAE,SAAS,EAAE,EACpB,eAAe,EAAE,0BAA0B,GAC1C,OAAO,CAAC,MAAM,CAAC,CAiCjB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE;IAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC;IAAC,OAAO,EAAE,SAAS,EAAE,CAAA;CAAC,EAC/D,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GACvC,IAAI,CAgFN;AAMD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAMpC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAwBF;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,GACb,MAAM,EAAE,CAKV;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B~~,8DAA8D~~;~~IAC9D~~,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;~~IAEpB~~;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,EACd,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,EAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,EACpC,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,EAClC,KAAK,CAAC,EAAE,aAAa,GAAG,SAAS,GAChC,kBAAkB,CAkCpB;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,cAAc,CAAC;CACxB,CAAC;AAEF,KAAK,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpC,KAAK,eAAe,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE5C,KAAK,sBAAsB,GAAG;IAC5B,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,UAAU,CAAC;CACxB,CAAC;AA4BF,eAAO,MAAM,uBAAuB,QAAkB,CAAC;AAoFvD,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI;oBAe7B,MAAM;;EASzB;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,sBAAsB;oBAXhD,MAAM;;EAmBzB;AA6GD,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,OAAO,GAClB,OAAO,CAAC,aAAa,CAAC,CA2CxB"}
1	+ {"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGzE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAoBzD,OAAO,KAAK,EAAC,SAAS,EAAE,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAaxE,OAAO,EAEL,KAAK,UAAU,EAGhB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAG1D,OAAO,EAEL,0BAA0B,EAC3B,MAAM,wCAAwC,CAAC;AAQhD,OAAO,EAML,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC/B,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9C,2BAA2B,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClD;;;;;;OAMG;IACH,MAAM,CAAC,EACH;QACE,6DAA6D;QAC7D,UAAU,EAAE,MAAM,CAAC;QACnB;;;WAGG;QACH,eAAe,EAAE,MAAM,CAAC;KACzB,GACD,SAAS,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,EAAC,KAAK,EAAE,QAAQ,EAAC,GAAE,cAA2C,GAC7D,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CA+RnC;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,aAAa,EACtB,WAAW,CAAC,EAAE,IAAI,CAAC,kBAAkB,EAAE,UAAU,CAAC,GACjD,OAAO,CAAC,IAAI,CAAC,CA+Bf;AAgKD;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,OAAO,EAAE,SAAS,EAAE,EACpB,eAAe,EAAE,0BAA0B,GAC1C,OAAO,CAAC,MAAM,CAAC,CAiCjB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE;IAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC;IAAC,OAAO,EAAE,SAAS,EAAE,CAAA;CAAC,EAC/D,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GACvC,IAAI,CAgFN;AAED;;;GAGG;AAEH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,kBAAkB,GAAG,MAAM,GAAG,IAAI,CAY7E;AAMD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAMpC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAwBF;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,GACb,MAAM,EAAE,CAKV;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEjC;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,EACd,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,EAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,EACpC,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,EAClC,KAAK,CAAC,EAAE,aAAa,GAAG,SAAS,GAChC,kBAAkB,CAkCpB;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,cAAc,CAAC;CACxB,CAAC;AAEF,KAAK,UAAU,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpC,KAAK,eAAe,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE5C,KAAK,sBAAsB,GAAG;IAC5B,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,UAAU,CAAC;CACxB,CAAC;AA4BF,eAAO,MAAM,uBAAuB,QAAkB,CAAC;AAoFvD,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI;oBAe7B,MAAM;;EASzB;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,sBAAsB;oBAXhD,MAAM;;EAmBzB;AA6GD,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,OAAO,GAClB,OAAO,CAAC,aAAa,CAAC,CA2CxB"}

package/out/zero-cache/src/services/change-source/pg/initial-sync.js CHANGED Viewed

@@ -26,7 +26,6 @@ import { BinaryCopyParser, hasBinaryDecoder, makeBinaryDecoder, textCastDecoder
 import { TsvParser } from "../../../db/pg-copy.js";
 import { getTypeParsers } from "../../../db/pg-type-parser.js";
 import { TransactionPool, importSnapshot } from "../../../db/transaction-pool.js";
-import { publicationRowFilter } from "./backfill-resume.js";
 import { createReplicaAndSlot } from "./replication-slots.js";
 import { ensureShardSchema } from "./schema/init.js";
 import { platform, tmpdir } from "node:os";
@@ -356,6 +355,16 @@ function verifyShadowReplica(lc, db, published, rowsByTable) {
 	if (issues.length) throw new Error(`Shadow replica verification failed (${issues.length} issue(s)):\n` + issues.map((i) => `  - ${i}`).join("\n"));
 	lc.info?.(`Shadow replica verification passed: ${published.tables.length} tables, ${published.indexes.length} indexes, ${columnsChecked} columns, ${rowsChecked.toLocaleString()} rows`);
 }
+/**
+* The row filter of the table's publications, as a parenthesized boolean
+* expression, or `null` if the table is published without a filter.
+*/
+function publicationRowFilter(table) {
+	const rowFilters = Object.values(table.publications).map(({ rowFilter }) => rowFilter);
+	if (rowFilters.some((filter) => filter === null)) return null;
+	const filters = rowFilters.filter((filter) => !!filter);
+	return filters.length === 0 ? null : `(${filters.join(" OR ")})`;
+}
 var MB = 1024 * 1024;
 var MAX_BUFFERED_ROWS = 1e4;
 var BUFFERED_SIZE_THRESHOLD = 8 * MB;
@@ -389,7 +398,7 @@ function makeDownloadStatements(table, cols, sampleRate, maxRowsPerTable, select
 	const where = conditions === null ? "" : `WHERE ${conditions}`;
 	const sample = tableSampleClause(sampleRate);
 	const limit = limitClause(maxRowsPerTable);
-	const orderBy = order ? ` ORDER BY ${order.by}` : "";
+	const orderBy = order?.by ? ` ORDER BY ${order.by}` : "";
 	const fromTable = `FROM ${id(table.schema)}.${id(table.name)}${sample} ${where}`;
 	const select = `SELECT ${(selectExprs ?? cols.map(id)).join(",")} ${fromTable}${orderBy}${limit}`;
 	if (limit) {