@rocicorp/zero 1.8.0-canary.2 → 1.8.0-canary.4

package/out/zero-cache/src/services/change-source/pg/initial-sync.js

@@ -8,6 +8,7 @@ import { liteTableName } from "../../../types/names.js";
 import { mapPostgresToLite, mapPostgresToLiteIndex } from "../../../db/pg-to-lite.js";
 import { ColumnMetadataStore } from "../../replicator/schema/column-metadata.js";
 import { listIndexes, listTables } from "../../../db/lite-tables.js";
+import { getOrCreateCounter, getOrCreateHistogram } from "../../../observability/metrics.js";
 import { initReplicationState } from "../../replicator/schema/replication-state.js";
 import { id } from "../../../types/sql.js";
 import { connectPgClient, pgClient } from "../../../types/pg.js";
@@ -36,17 +37,22 @@ import { pipeline as pipeline$1 } from "node:stream/promises";
 async function initialSync(lc, shard, tx, upstreamURI, syncOptions, context) {
 	if (!ALLOWED_APP_ID_CHARACTERS.test(shard.appID)) throw new Error("The App ID may only consist of lower-case letters, numbers, and the underscore character");
 	const { tableCopyWorkers, profileCopy, textCopy = false, replicationSlotFailover = false, shadow } = syncOptions;
-	const copyProfiler = profileCopy ? await CpuProfiler.connect() : null;
-	const sql = await connectPgClient(lc, upstreamURI, "initial-sync");
-	const replicationSession = shadow ? void 0 : pgClient(lc, upstreamURI, "initial-sync-replication-session", {
-		["fetch_types"]: false,
-		connection: { replication: "database" }
-	});
+	const syncMode = shadow ? "shadow" : "initial";
+	const copyFormat = textCopy ? "text" : "binary";
+	const start = performance.now();
+	let sql;
+	let replicationSession;
 	const replicaID = Date.now().toString();
 	let slotName;
 	const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx, shadow ? async () => {} : void 0).publish(lc, "Initializing");
 	let releaseShadowSnapshot;
 	try {
+		const copyProfiler = profileCopy ? await CpuProfiler.connect() : null;
+		sql = await connectPgClient(lc, upstreamURI, "initial-sync");
+		replicationSession = shadow ? void 0 : pgClient(lc, upstreamURI, "initial-sync-replication-session", {
+			["fetch_types"]: false,
+			connection: { replication: "database" }
+		});
 		const pgVersion = await checkUpstreamConfig(sql);
 		const { publications } = shadow ? await getInternalShardConfig(sql, shard) : await ensurePublishedTables(lc, sql, shard);
 		lc.info?.(`Upstream is setup with publications [${publications}]`);
@@ -67,7 +73,6 @@ async function initialSync(lc, shard, tx, upstreamURI, syncOptions, context) {
 		}
 		const initialVersion = toStateVersionString(lsn);
 		initReplicationState(tx, publications, initialVersion, context);
-		const start = performance.now();
 		const published = await runTx(sql, async (tx) => {
 			await tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshot}'`);
 			return getPublicationInfo(tx, publications);
@@ -89,16 +94,12 @@ async function initialSync(lc, shard, tx, upstreamURI, syncOptions, context) {
 			const downloads = await Promise.all(tables.map((spec) => copiers.processReadTask((db, lc) => getInitialDownloadState(lc, db, spec, shadow !== void 0))));
 			statusPublisher.publish(lc, "Initializing", `Copying ${numTables} upstream tables at version ${initialVersion}`, 5e3, () => ({ downloadStatus: downloads.map(({ status }) => status) }));
 			copyProfiler?.start();
-			const rowCounts = await Promise.all(downloads.map((table) => copiers.processReadTask((db, lc) => copy(lc, table, copyPool, db, tx, textCopy, sampleRate, maxRowsPerTable))));
+			const copyStart = performance.now();
+			const copyResults = await Promise.all(downloads.map((table) => copiers.processReadTask((db, lc) => copy(lc, table, copyPool, db, tx, textCopy, syncMode, sampleRate, maxRowsPerTable))));
+			const copyElapsed = performance.now() - copyStart;
 			copyProfiler?.stopAndDispose(lc, "initial-copy");
 			copiers.setDone();
-			const total = rowCounts.reduce((acc, curr) => ({
-				rows: acc.rows + curr.rows,
-				flushTime: acc.flushTime + curr.flushTime
-			}), {
-				rows: 0,
-				flushTime: 0
-			});
+			const copySummary = initialSyncCopySummary(copyResults, copyElapsed);
 			statusPublisher.publish(lc, "Indexing", `Creating ${indexes.length} indexes`, 5e3);
 			const indexStart = performance.now();
 			createLiteIndices(lc, tx, indexes);
@@ -108,16 +109,45 @@ async function initialSync(lc, shard, tx, upstreamURI, syncOptions, context) {
 			else {
 				assert(shadow, "expected to be in shadow sync if there is no slotName");
 				const rowsByTable = /* @__PURE__ */ new Map();
-				for (let i = 0; i < downloads.length; i++) rowsByTable.set(downloads[i].status.table, rowCounts[i].rows);
+				for (let i = 0; i < downloads.length; i++) rowsByTable.set(downloads[i].status.table, copyResults[i].rows);
 				verifyShadowReplica(lc, tx, published, rowsByTable);
 			}
 			const elapsed = performance.now() - start;
-			lc.info?.(`Synced ${total.rows.toLocaleString()} rows of ${numTables} tables in ${publications} up to ${lsn} (flush: ${total.flushTime.toFixed(3)}, index: ${index.toFixed(3)}, total: ${elapsed.toFixed(3)} ms)`);
+			const copyOtherMs = Math.max(0, elapsed - copySummary.flushMs - index);
+			recordInitialSyncRunMetrics({
+				durationMs: elapsed,
+				rows: copySummary.rows,
+				copyBytes: copySummary.copyBytes,
+				copyMs: copySummary.copyMs,
+				copyOtherMs,
+				flushMs: copySummary.flushMs,
+				indexMs: index
+			}, {
+				result: "success",
+				syncMode,
+				copyFormat
+			});
+			lc.info?.(`Synced ${copySummary.rows.toLocaleString()} rows of ${numTables} tables in ${publications} up to ${lsn} (flush: ${copySummary.flushMs.toFixed(3)}, index: ${index.toFixed(3)}, total: ${elapsed.toFixed(3)} ms)`, {
+				syncMode,
+				copyFormat,
+				publications,
+				lsn,
+				...copySummary,
+				indexes: indexes.length,
+				indexMs: index,
+				copyOtherMs,
+				totalMs: elapsed
+			});
 		} finally {
 			copyPool.end().catch((e) => lc.warn?.(`Error closing copyPool`, e));
 		}
 	} catch (e) {
-		if (slotName) {
+		recordInitialSyncRunMetrics({ durationMs: performance.now() - start }, {
+			result: "error",
+			syncMode,
+			copyFormat
+		});
+		if (slotName && sql) {
 			lc.warn?.(`dropping replication slot ${slotName}`, e);
 			await sql`
         SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots
@@ -129,7 +159,7 @@ async function initialSync(lc, shard, tx, upstreamURI, syncOptions, context) {
 		statusPublisher.stop();
 		if (releaseShadowSnapshot) await releaseShadowSnapshot().catch((e) => lc.warn?.(`Error releasing shadow snapshot`, e));
 		if (replicationSession) await replicationSession.end();
-		await sql.end();
+		if (sql) await sql.end();
 	}
 }
 /**
@@ -368,6 +398,113 @@ function makeDownloadStatements(table, cols, sampleRate, maxRowsPerTable, select
 		getTotalBytes: `SELECT ${totalBytes} AS "totalBytes" ${fromTable}`
 	};
 }
+var INITIAL_SYNC_DURATION_HISTOGRAM_BOUNDARIES_S = [
+	1,
+	2,
+	5,
+	10,
+	30,
+	60,
+	120,
+	300,
+	600,
+	1200,
+	2400,
+	3600,
+	7200
+];
+var SLOW_COPY_FLUSH_MS = 1e4;
+function initialSyncRuns() {
+	return getOrCreateCounter("replication", "initial_sync_runs", "Initial sync runs, labeled by result.");
+}
+function initialSyncDuration() {
+	return initialSyncDurationHistogram("initial_sync_duration", "Wall-clock duration of an initial sync run, labeled by result.");
+}
+function initialSyncCopyDuration() {
+	return initialSyncDurationHistogram("initial_sync_copy_duration", "Wall-clock duration of the COPY phase for a successful initial sync run.");
+}
+function initialSyncCopyOtherDuration() {
+	return initialSyncDurationHistogram("initial_sync_copy_other_duration", "Initial sync total duration excluding SQLite flush and index time for a successful run.");
+}
+function initialSyncFlushDuration() {
+	return initialSyncDurationHistogram("initial_sync_flush_duration", "Total SQLite flush time for a successful initial sync run.");
+}
+function initialSyncIndexDuration() {
+	return initialSyncDurationHistogram("initial_sync_index_duration", "SQLite index creation time for a successful initial sync run.");
+}
+function initialSyncRows() {
+	return getOrCreateCounter("replication", "initial_sync_rows", "Rows copied during successful initial sync runs.");
+}
+function initialSyncCopyStream() {
+	return getOrCreateCounter("replication", "initial_sync_copy_stream", {
+		description: "PostgreSQL COPY stream bytes processed during initial sync, including in-progress and failed runs.",
+		unit: "bytes"
+	});
+}
+function initialSyncCompletedCopyStream() {
+	return getOrCreateCounter("replication", "initial_sync_completed_copy_stream", {
+		description: "PostgreSQL COPY stream bytes processed during successful initial sync runs.",
+		unit: "bytes"
+	});
+}
+function initialSyncCopyChunks() {
+	return getOrCreateCounter("replication", "initial_sync_copy_chunks", "PostgreSQL COPY stream chunks processed during initial sync.");
+}
+function initialSyncDurationHistogram(name, description) {
+	return getOrCreateHistogram("replication", name, {
+		description,
+		unit: "s",
+		bucketBoundaries: INITIAL_SYNC_DURATION_HISTOGRAM_BOUNDARIES_S
+	});
+}
+function initialSyncMetricAttrs(attrs) {
+	return {
+		sync_mode: attrs.syncMode,
+		copy_format: attrs.copyFormat
+	};
+}
+function initialSyncRunMetricAttrs(attrs) {
+	return {
+		...initialSyncMetricAttrs(attrs),
+		result: attrs.result
+	};
+}
+function recordInitialSyncRunMetrics(stats, attrs) {
+	const labels = initialSyncRunMetricAttrs(attrs);
+	initialSyncRuns().add(1, labels);
+	initialSyncDuration().recordMs(stats.durationMs, labels);
+	if (attrs.result === "success") {
+		if (stats.copyMs !== void 0) initialSyncCopyDuration().recordMs(stats.copyMs, labels);
+		if (stats.copyOtherMs !== void 0) initialSyncCopyOtherDuration().recordMs(stats.copyOtherMs, labels);
+		if (stats.flushMs !== void 0) initialSyncFlushDuration().recordMs(stats.flushMs, labels);
+		if (stats.indexMs !== void 0) initialSyncIndexDuration().recordMs(stats.indexMs, labels);
+		if (stats.rows !== void 0 && stats.rows > 0) initialSyncRows().add(stats.rows, labels);
+		if (stats.copyBytes !== void 0 && stats.copyBytes > 0) initialSyncCompletedCopyStream().add(stats.copyBytes, labels);
+	}
+}
+function initialSyncCopySummary(results, copyMs) {
+	const totals = results.reduce((acc, curr) => {
+		acc.rows += curr.rows;
+		acc.flushMs += curr.flushMs;
+		acc.copyBytes += curr.copyBytes;
+		return acc;
+	}, {
+		rows: 0,
+		flushMs: 0,
+		copyBytes: 0
+	});
+	return {
+		tables: results.length,
+		rows: totals.rows,
+		copyMs,
+		flushMs: totals.flushMs,
+		copyBytes: totals.copyBytes
+	};
+}
+function logSlowCopyFlush(lc, details) {
+	if (details.elapsedMs < SLOW_COPY_FLUSH_MS || details.flushedRows === 0) return;
+	lc.info?.("initial-sync table copy slow flush", details);
+}
 async function getInitialDownloadState(lc, sql, spec, skipTotals) {
 	const start = performance.now();
 	const table = liteTableName(spec);
@@ -403,13 +540,21 @@ async function getInitialDownloadState(lc, sql, spec, skipTotals) {
 	lc.info?.(`Computed initial download state for ${table} (${elapsed} ms)`, { state: state.status });
 	return state;
 }
-function copy(lc, { spec: table, status }, dbClient, from, to, textCopy, sampleRate, maxRowsPerTable) {
-	if (textCopy) return copyText(lc, table, status, dbClient, from, to, sampleRate, maxRowsPerTable);
-	return copyBinary(lc, table, status, from, to, sampleRate, maxRowsPerTable);
+function copy(lc, { spec: table, status }, dbClient, from, to, textCopy, syncMode, sampleRate, maxRowsPerTable) {
+	if (textCopy) return copyText(lc, table, status, dbClient, from, to, syncMode, sampleRate, maxRowsPerTable);
+	return copyBinary(lc, table, status, from, to, syncMode, sampleRate, maxRowsPerTable);
 }
-async function copyBinary(lc, table, status, from, to, sampleRate, maxRowsPerTable) {
+async function copyBinary(lc, table, status, from, to, syncMode, sampleRate, maxRowsPerTable) {
 	const start = performance.now();
-	let flushTime = 0;
+	const copyFormat = "binary";
+	const metricLabels = initialSyncMetricAttrs({
+		syncMode,
+		copyFormat
+	});
+	const copyStreamMetric = initialSyncCopyStream();
+	const copyChunksMetric = initialSyncCopyChunks();
+	let flushMs = 0;
+	let copyBytes = 0;
 	const tableName = liteTableName(table);
 	const orderedColumns = Object.entries(table.columns);
 	const columnNames = orderedColumns.map(([c]) => c);
@@ -428,7 +573,7 @@ async function copyBinary(lc, table, status, from, to, sampleRate, maxRowsPerTab
 	let pendingRows = 0;
 	let pendingSize = 0;
 	function flush() {
-		const start = performance.now();
+		const flushStart = performance.now();
 		const flushedRows = pendingRows;
 		const flushedSize = pendingSize;
 		let l = 0;
@@ -438,9 +583,21 @@ async function copyBinary(lc, table, status, from, to, sampleRate, maxRowsPerTab
 		for (let i = 0; i < flushedValues; i++) pendingValues[i] = void 0;
 		pendingSize = 0;
 		status.rows += flushedRows;
-		const elapsed = performance.now() - start;
-		flushTime += elapsed;
+		const elapsed = performance.now() - flushStart;
+		flushMs += elapsed;
 		lc.debug?.(`flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`);
+		logSlowCopyFlush(lc, {
+			schema: table.schema,
+			table: table.name,
+			replicaTable: tableName,
+			syncMode,
+			copyFormat,
+			elapsedMs: elapsed,
+			flushedRows,
+			flushedBytes: flushedSize,
+			rows: status.rows,
+			copyBytes
+		});
 	}
 	const binaryParser = new BinaryCopyParser();
 	let col = 0;
@@ -449,8 +606,12 @@ async function copyBinary(lc, table, status, from, to, sampleRate, maxRowsPerTab
 		highWaterMark: BUFFERED_SIZE_THRESHOLD,
 		write(chunk, _encoding, callback) {
 			try {
+				copyBytes += chunk.length;
+				copyStreamMetric.add(chunk.length, metricLabels);
+				copyChunksMetric.add(1, metricLabels);
 				for (const fieldBuf of binaryParser.parse(chunk)) {
-					pendingSize += fieldBuf === null ? 4 : fieldBuf.length;
+					const fieldSize = fieldBuf === null ? 4 : fieldBuf.length;
+					pendingSize += fieldSize;
 					pendingValues[pendingRows * valuesPerRow + col] = fieldBuf === null ? null : decoders[col](fieldBuf);
 					if (++col === decoders.length) {
 						col = 0;
@@ -472,15 +633,34 @@ async function copyBinary(lc, table, status, from, to, sampleRate, maxRowsPerTab
 		}
 	}));
 	const elapsed = performance.now() - start;
-	lc.info?.(`Finished copying ${status.rows} rows into ${tableName} (flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `);
-	return {
+	const result = {
+		schema: table.schema,
+		table: table.name,
+		replicaTable: tableName,
+		syncMode,
+		copyFormat,
+		columnCount: columnNames.length,
 		rows: status.rows,
-		flushTime
+		estimatedRows: status.totalRows,
+		estimatedBytes: status.totalBytes,
+		flushMs,
+		elapsedMs: elapsed,
+		copyBytes
 	};
+	lc.info?.(`Finished copying ${status.rows} rows into ${tableName} (flush: ${flushMs.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `, result);
+	return result;
 }
-async function copyText(lc, table, status, dbClient, from, to, sampleRate, maxRowsPerTable) {
+async function copyText(lc, table, status, dbClient, from, to, syncMode, sampleRate, maxRowsPerTable) {
 	const start = performance.now();
-	let flushTime = 0;
+	const copyFormat = "text";
+	const metricLabels = initialSyncMetricAttrs({
+		syncMode,
+		copyFormat
+	});
+	const copyStreamMetric = initialSyncCopyStream();
+	const copyChunksMetric = initialSyncCopyChunks();
+	let flushMs = 0;
+	let copyBytes = 0;
 	const tableName = liteTableName(table);
 	const orderedColumns = Object.entries(table.columns);
 	const columnNames = orderedColumns.map(([c]) => c);
@@ -498,7 +678,7 @@ async function copyText(lc, table, status, dbClient, from, to, sampleRate, maxRo
 	let pendingRows = 0;
 	let pendingSize = 0;
 	function flush() {
-		const start = performance.now();
+		const flushStart = performance.now();
 		const flushedRows = pendingRows;
 		const flushedSize = pendingSize;
 		let l = 0;
@@ -508,9 +688,21 @@ async function copyText(lc, table, status, dbClient, from, to, sampleRate, maxRo
 		for (let i = 0; i < flushedValues; i++) pendingValues[i] = void 0;
 		pendingSize = 0;
 		status.rows += flushedRows;
-		const elapsed = performance.now() - start;
-		flushTime += elapsed;
+		const elapsed = performance.now() - flushStart;
+		flushMs += elapsed;
 		lc.debug?.(`flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`);
+		logSlowCopyFlush(lc, {
+			schema: table.schema,
+			table: table.name,
+			replicaTable: tableName,
+			syncMode,
+			copyFormat,
+			elapsedMs: elapsed,
+			flushedRows,
+			flushedBytes: flushedSize,
+			rows: status.rows,
+			copyBytes
+		});
 	}
 	lc.info?.(`Starting text copy stream of ${tableName}:`, select);
 	const pgParsers = await getTypeParsers(dbClient, { returnJsonAsString: true });
@@ -524,8 +716,12 @@ async function copyText(lc, table, status, dbClient, from, to, sampleRate, maxRo
 		highWaterMark: BUFFERED_SIZE_THRESHOLD,
 		write(chunk, _encoding, callback) {
 			try {
+				copyBytes += chunk.length;
+				copyStreamMetric.add(chunk.length, metricLabels);
+				copyChunksMetric.add(1, metricLabels);
 				for (const text of tsvParser.parse(chunk)) {
-					pendingSize += text === null ? 4 : text.length;
+					const fieldSize = text === null ? 4 : text.length;
+					pendingSize += fieldSize;
 					pendingValues[pendingRows * valuesPerRow + col] = text === null ? null : parsers[col](text);
 					if (++col === parsers.length) {
 						col = 0;
@@ -547,11 +743,22 @@ async function copyText(lc, table, status, dbClient, from, to, sampleRate, maxRo
 		}
 	}));
 	const elapsed = performance.now() - start;
-	lc.info?.(`Finished copying ${status.rows} rows into ${tableName} (flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `);
-	return {
+	const result = {
+		schema: table.schema,
+		table: table.name,
+		replicaTable: tableName,
+		syncMode,
+		copyFormat,
+		columnCount: columnNames.length,
 		rows: status.rows,
-		flushTime
+		estimatedRows: status.totalRows,
+		estimatedBytes: status.totalBytes,
+		flushMs,
+		elapsedMs: elapsed,
+		copyBytes
 	};
+	lc.info?.(`Finished copying ${status.rows} rows into ${tableName} (flush: ${flushMs.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `, result);
+	return result;
 }
 //#endregion
 export { initialSync, makeBinarySelectExprs, makeDownloadStatements, shadowInitialSync };