@rocicorp/zero 1.2.0-canary.1 → 1.2.0-canary.12

package/out/zero-cache/src/services/change-source/pg/initial-sync.js

@@ -18,6 +18,7 @@ import { ReplicationStatusPublisher } from "../../replicator/replication-status.
 import { TsvParser } from "../../../db/pg-copy.js";
 import { getTypeParsers } from "../../../db/pg-type-parser.js";
 import { TransactionPool, importSnapshot } from "../../../db/transaction-pool.js";
+import { BinaryCopyParser, hasBinaryDecoder, makeBinaryDecoder, textCastDecoder } from "../../../db/pg-copy-binary.js";
 import { CpuProfiler } from "../../../types/profiler.js";
 import { ensureShardSchema } from "./schema/init.js";
 import postgres from "postgres";
@@ -28,7 +29,7 @@ import { pipeline as pipeline$1 } from "node:stream/promises";
 //#region ../zero-cache/src/services/change-source/pg/initial-sync.ts
 async function initialSync(lc, shard, tx, upstreamURI, syncOptions, context) {
 	if (!ALLOWED_APP_ID_CHARACTERS.test(shard.appID)) throw new Error("The App ID may only consist of lower-case letters, numbers, and the underscore character");
-	const { tableCopyWorkers, profileCopy } = syncOptions;
+	const { tableCopyWorkers, profileCopy, textCopy = false } = syncOptions;
 	const copyProfiler = profileCopy ? await CpuProfiler.connect() : null;
 	const sql = pgClient(lc, upstreamURI);
 	const replicationSession = pgClient(lc, upstreamURI, {
@@ -92,7 +93,7 @@ async function initialSync(lc, shard, tx, upstreamURI, syncOptions, context) {
 			const downloads = await Promise.all(tables.map((spec) => copiers.processReadTask((db, lc) => getInitialDownloadState(lc, db, spec))));
 			statusPublisher.publish(lc, "Initializing", `Copying ${numTables} upstream tables at version ${initialVersion}`, 5e3, () => ({ downloadStatus: downloads.map(({ status }) => status) }));
 			copyProfiler?.start();
-			const rowCounts = await Promise.all(downloads.map((table) => copiers.processReadTask((db, lc) => copy(lc, table, copyPool, db, tx))));
+			const rowCounts = await Promise.all(downloads.map((table) => copiers.processReadTask((db, lc) => copy(lc, table, copyPool, db, tx, textCopy))));
 			copyProfiler?.stopAndDispose(lc, "initial-copy");
 			copiers.setDone();
 			const total = rowCounts.reduce((acc, curr) => ({
@@ -214,7 +215,85 @@ async function getInitialDownloadState(lc, sql, spec) {
 	lc.info?.(`Computed initial download state for ${table} (${elapsed} ms)`, { state: state.status });
 	return state;
 }
-async function copy(lc, { spec: table, status }, dbClient, from, to) {
+function copy(lc, { spec: table, status }, dbClient, from, to, textCopy) {
+	if (textCopy) return copyText(lc, table, status, dbClient, from, to);
+	return copyBinary(lc, table, status, from, to);
+}
+async function copyBinary(lc, table, status, from, to) {
+	const start = performance.now();
+	let flushTime = 0;
+	const tableName = liteTableName(table);
+	const orderedColumns = Object.entries(table.columns);
+	const columnNames = orderedColumns.map(([c]) => c);
+	const columnSpecs = orderedColumns.map(([_name, spec]) => spec);
+	const insertColumnList = columnNames.map((c) => id(c)).join(",");
+	const valuesSql = columnNames.length > 0 ? `(${"?,".repeat(columnNames.length - 1)}?)` : "()";
+	const insertSql = `
+    INSERT INTO "${tableName}" (${insertColumnList}) VALUES ${valuesSql}`;
+	const insertStmt = to.prepare(insertSql);
+	const insertBatchStmt = to.prepare(insertSql + `,${valuesSql}`.repeat(49));
+	const filterConditions = Object.values(table.publications).map(({ rowFilter }) => rowFilter).filter((f) => !!f);
+	const where = filterConditions.length === 0 ? "" : `WHERE ${filterConditions.join(" OR ")}`;
+	const fromTable = `FROM ${id(table.schema)}.${id(table.name)} ${where}`;
+	const select = `SELECT ${orderedColumns.map(([name, spec]) => hasBinaryDecoder(spec) ? id(name) : `${id(name)}::text`).join(",")} ${fromTable}`;
+	const decoders = orderedColumns.map(([, spec]) => hasBinaryDecoder(spec) ? makeBinaryDecoder(spec) : textCastDecoder);
+	const valuesPerRow = columnSpecs.length;
+	const valuesPerBatch = valuesPerRow * 50;
+	const pendingValues = Array.from({ length: MAX_BUFFERED_ROWS * valuesPerRow });
+	let pendingRows = 0;
+	let pendingSize = 0;
+	function flush() {
+		const start = performance.now();
+		const flushedRows = pendingRows;
+		const flushedSize = pendingSize;
+		let l = 0;
+		for (; pendingRows > 50; pendingRows -= 50) insertBatchStmt.run(pendingValues.slice(l, l += valuesPerBatch));
+		for (; pendingRows > 0; pendingRows--) insertStmt.run(pendingValues.slice(l, l += valuesPerRow));
+		const flushedValues = flushedRows * valuesPerRow;
+		for (let i = 0; i < flushedValues; i++) pendingValues[i] = void 0;
+		pendingSize = 0;
+		status.rows += flushedRows;
+		const elapsed = performance.now() - start;
+		flushTime += elapsed;
+		lc.debug?.(`flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`);
+	}
+	const binaryParser = new BinaryCopyParser();
+	let col = 0;
+	lc.info?.(`Starting binary copy stream of ${tableName}:`, select);
+	await pipeline$1(await from.unsafe(`COPY (${select}) TO STDOUT WITH (FORMAT binary)`).readable(), new Writable({
+		highWaterMark: BUFFERED_SIZE_THRESHOLD,
+		write(chunk, _encoding, callback) {
+			try {
+				for (const fieldBuf of binaryParser.parse(chunk)) {
+					pendingSize += fieldBuf === null ? 4 : fieldBuf.length;
+					pendingValues[pendingRows * valuesPerRow + col] = fieldBuf === null ? null : decoders[col](fieldBuf);
+					if (++col === decoders.length) {
+						col = 0;
+						if (++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow || pendingSize >= BUFFERED_SIZE_THRESHOLD) flush();
+					}
+				}
+				callback();
+			} catch (e) {
+				callback(e instanceof Error ? e : new Error(String(e)));
+			}
+		},
+		final: (callback) => {
+			try {
+				flush();
+				callback();
+			} catch (e) {
+				callback(e instanceof Error ? e : new Error(String(e)));
+			}
+		}
+	}));
+	const elapsed = performance.now() - start;
+	lc.info?.(`Finished copying ${status.rows} rows into ${tableName} (flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `);
+	return {
+		rows: status.rows,
+		flushTime
+	};
+}
+async function copyText(lc, table, status, dbClient, from, to) {
 	const start = performance.now();
 	let flushTime = 0;
 	const tableName = liteTableName(table);
@@ -240,14 +319,15 @@ async function copy(lc, { spec: table, status }, dbClient, from, to) {
 		let l = 0;
 		for (; pendingRows > 50; pendingRows -= 50) insertBatchStmt.run(pendingValues.slice(l, l += valuesPerBatch));
 		for (; pendingRows > 0; pendingRows--) insertStmt.run(pendingValues.slice(l, l += valuesPerRow));
-		for (let i = 0; i < flushedRows; i++) pendingValues[i] = void 0;
+		const flushedValues = flushedRows * valuesPerRow;
+		for (let i = 0; i < flushedValues; i++) pendingValues[i] = void 0;
 		pendingSize = 0;
 		status.rows += flushedRows;
 		const elapsed = performance.now() - start;
 		flushTime += elapsed;
 		lc.debug?.(`flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`);
 	}
-	lc.info?.(`Starting copy stream of ${tableName}:`, select);
+	lc.info?.(`Starting text copy stream of ${tableName}:`, select);
 	const pgParsers = await getTypeParsers(dbClient, { returnJsonAsString: true });
 	const parsers = columnSpecs.map((c) => {
 		const pgParse = pgParsers.getTypeParser(c.typeOID);

package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map

@@ -1 +1 @@
-{"version":3,"file":"initial-sync.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"sourcesContent":["import {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {platform} from 'node:os';\nimport {Writable} from 'node:stream';\nimport {pipeline} from 'node:stream/promises';\nimport postgres from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport type {DownloadStatus} from '../../../../../zero-events/src/status.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {\n  createLiteIndexStatement,\n  createLiteTableStatement,\n} from '../../../db/create.ts';\nimport * as Mode from '../../../db/mode-enum.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {\n  mapPostgresToLite,\n  mapPostgresToLiteIndex,\n} from '../../../db/pg-to-lite.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport type {IndexSpec, PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {\n  JSON_STRINGIFIED,\n  liteValue,\n  type LiteValueType,\n} from '../../../types/lite.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport {\n  pgClient,\n  type PostgresDB,\n  type PostgresTransaction,\n  type PostgresValueType,\n} from '../../../types/pg.ts';\nimport {CpuProfiler} from '../../../types/profiler.ts';\nimport type {ShardConfig} from '../../../types/shards.ts';\nimport {ALLOWED_APP_ID_CHARACTERS} from '../../../types/shards.ts';\nimport {id} from '../../../types/sql.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {ColumnMetadataStore} from '../../replicator/schema/column-metadata.ts';\nimport {initReplicationState} from '../../replicator/schema/replication-state.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport {\n  addReplica,\n  dropShard,\n  getInternalShardConfig,\n  newReplicationSlot,\n  replicationSlotExpression,\n  validatePublications,\n} from './schema/shard.ts';\n\nexport type InitialSyncOptions = {\n  tableCopyWorkers: number;\n  profileCopy?: boolean | undefined;\n};\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n) {\n  if (!ALLOWED_APP_ID_CHARACTERS.test(shard.appID)) {\n    throw new Error(\n      'The App ID may only consist of lower-case letters, numbers, and the underscore character',\n    );\n  }\n  const {tableCopyWorkers, profileCopy} = syncOptions;\n  const copyProfiler = profileCopy ? await CpuProfiler.connect() : null;\n  const sql = pgClient(lc, upstreamURI);\n  const replicationSession = pgClient(lc, upstreamURI, {\n    ['fetch_types']: false, // Necessary for the streaming protocol\n    connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n  });\n  const slotName = newReplicationSlot(shard);\n  const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(\n    tx,\n  ).publish(lc, 'Initializing');\n  try {\n    await checkUpstreamConfig(sql);\n\n    const {publications} = await ensurePublishedTables(lc, sql, shard);\n    lc.info?.(`Upstream is setup with publications [${publications}]`);\n\n    const {database, host} = sql.options;\n    lc.info?.(`opening replication session to ${database}@${host}`);\n\n    let slot: ReplicationSlot;\n    for (let first = true; ; first = false) {\n      try {\n        slot = await createReplicationSlot(lc, replicationSession, slotName);\n        break;\n      } catch (e) {\n        if (first && e instanceof postgres.PostgresError) {\n          if (e.code === PG_INSUFFICIENT_PRIVILEGE) {\n            // Some Postgres variants (e.g. Google Cloud SQL) require that\n            // the user have the REPLICATION role in order to create a slot.\n            // Note that this must be done by the upstreamDB connection, and\n            // does not work in the replicationSession itself.\n            await sql`ALTER ROLE current_user WITH REPLICATION`;\n            lc.info?.(`Added the REPLICATION role to database user`);\n            continue;\n          }\n          if (e.code === PG_CONFIGURATION_LIMIT_EXCEEDED) {\n            const slotExpression = replicationSlotExpression(shard);\n\n            const dropped = await sql<{slot: string}[]>`\n              SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n                FROM pg_replication_slots\n                WHERE slot_name LIKE ${slotExpression} AND NOT active`;\n            if (dropped.length) {\n              lc.warn?.(\n                `Dropped inactive replication slots: ${dropped.map(({slot}) => slot)}`,\n                e,\n              );\n              continue;\n            }\n            lc.error?.(`Unable to drop replication slots`, e);\n          }\n        }\n        throw e;\n      }\n    }\n    const {snapshot_name: snapshot, consistent_point: lsn} = slot;\n    const initialVersion = toStateVersionString(lsn);\n\n    initReplicationState(tx, publications, initialVersion, context);\n\n    // Run up to MAX_WORKERS to copy of tables at the replication slot's snapshot.\n    const start = performance.now();\n    // Retrieve the published schema at the consistent_point.\n    const published = await runTx(\n      sql,\n      async tx => {\n        await tx.unsafe(/* sql*/ `SET TRANSACTION SNAPSHOT '${snapshot}'`);\n        return getPublicationInfo(tx, publications);\n      },\n      {mode: Mode.READONLY},\n    );\n    // Note: If this throws, initial-sync is aborted.\n    validatePublications(lc, published);\n\n    // Now that tables have been validated, kick off the copiers.\n    const {tables, indexes} = published;\n    const numTables = tables.length;\n    if (platform() === 'win32' && tableCopyWorkers < numTables) {\n      lc.warn?.(\n        `Increasing the number of copy workers from ${tableCopyWorkers} to ` +\n          `${numTables} to work around a Node/Postgres connection bug`,\n      );\n    }\n    const numWorkers =\n      platform() === 'win32'\n        ? numTables\n        : Math.min(tableCopyWorkers, numTables);\n\n    const copyPool = pgClient(lc, upstreamURI, {\n      max: numWorkers,\n      connection: {['application_name']: 'initial-sync-copy-worker'},\n      ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n    });\n    const copiers = startTableCopyWorkers(\n      lc,\n      copyPool,\n      snapshot,\n      numWorkers,\n      numTables,\n    );\n    try {\n      createLiteTables(tx, tables, initialVersion);\n      const downloads = await Promise.all(\n        tables.map(spec =>\n          copiers.processReadTask((db, lc) =>\n            getInitialDownloadState(lc, db, spec),\n          ),\n        ),\n      );\n      statusPublisher.publish(\n        lc,\n        'Initializing',\n        `Copying ${numTables} upstream tables at version ${initialVersion}`,\n        5000,\n        () => ({downloadStatus: downloads.map(({status}) => status)}),\n      );\n\n      void copyProfiler?.start();\n      const rowCounts = await Promise.all(\n        downloads.map(table =>\n          copiers.processReadTask((db, lc) =>\n            copy(lc, table, copyPool, db, tx),\n          ),\n        ),\n      );\n      void copyProfiler?.stopAndDispose(lc, 'initial-copy');\n      copiers.setDone();\n\n      const total = rowCounts.reduce(\n        (acc, curr) => ({\n          rows: acc.rows + curr.rows,\n          flushTime: acc.flushTime + curr.flushTime,\n        }),\n        {rows: 0, flushTime: 0},\n      );\n\n      statusPublisher.publish(\n        lc,\n        'Indexing',\n        `Creating ${indexes.length} indexes`,\n        5000,\n      );\n      const indexStart = performance.now();\n      createLiteIndices(tx, indexes);\n      const index = performance.now() - indexStart;\n      lc.info?.(`Created indexes (${index.toFixed(3)} ms)`);\n\n      await addReplica(\n        sql,\n        shard,\n        slotName,\n        initialVersion,\n        published,\n        context,\n      );\n\n      const elapsed = performance.now() - start;\n      lc.info?.(\n        `Synced ${total.rows.toLocaleString()} rows of ${numTables} tables in ${publications} up to ${lsn} ` +\n          `(flush: ${total.flushTime.toFixed(3)}, index: ${index.toFixed(3)}, total: ${elapsed.toFixed(3)} ms)`,\n      );\n    } finally {\n      // All meaningful errors are handled at the processReadTask() call site.\n      void copyPool.end().catch(e => lc.warn?.(`Error closing copyPool`, e));\n    }\n  } catch (e) {\n    // If initial-sync did not succeed, make a best effort to drop the\n    // orphaned replication slot to avoid running out of slots in\n    // pathological cases that result in repeated failures.\n    lc.warn?.(`dropping replication slot ${slotName}`, e);\n    await sql`\n      SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n        WHERE slot_name = ${slotName};\n    `.catch(e => lc.warn?.(`Unable to drop replication slot ${slotName}`, e));\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n    await replicationSession.end();\n    await sql.end();\n  }\n}\n\nasync function checkUpstreamConfig(sql: PostgresDB) {\n  const {walLevel, version} = (\n    await sql<{walLevel: string; version: number}[]>`\n      SELECT current_setting('wal_level') as \"walLevel\", \n             current_setting('server_version_num') as \"version\";\n  `\n  )[0];\n\n  if (walLevel !== 'logical') {\n    throw new Error(\n      `Postgres must be configured with \"wal_level = logical\" (currently: \"${walLevel})`,\n    );\n  }\n  if (version < 150000) {\n    throw new Error(\n      `Must be running Postgres 15 or higher (currently: \"${version}\")`,\n    );\n  }\n}\n\nasync function ensurePublishedTables(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  validate = true,\n): Promise<{publications: string[]}> {\n  const {database, host} = sql.options;\n  lc.info?.(`Ensuring upstream PUBLICATION on ${database}@${host}`);\n\n  await ensureShardSchema(lc, sql, shard);\n  const {publications} = await getInternalShardConfig(sql, shard);\n\n  if (validate) {\n    let valid = false;\n    const nonInternalPublications = publications.filter(\n      p => !p.startsWith('_'),\n    );\n    const exists = await sql`\n      SELECT pubname FROM pg_publication WHERE pubname IN ${sql(publications)}\n      `.values();\n    if (exists.length !== publications.length) {\n      lc.warn?.(\n        `some configured publications [${publications}] are missing: ` +\n          `[${exists.flat()}]. resyncing`,\n      );\n    } else if (\n      !equals(new Set(shard.publications), new Set(nonInternalPublications))\n    ) {\n      lc.warn?.(\n        `requested publications [${shard.publications}] differ from previous` +\n          `publications [${nonInternalPublications}]. resyncing`,\n      );\n    } else {\n      valid = true;\n    }\n    if (!valid) {\n      await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n      return ensurePublishedTables(lc, sql, shard, false);\n    }\n  }\n  return {publications};\n}\n\nfunction startTableCopyWorkers(\n  lc: LogContext,\n  db: PostgresDB,\n  snapshot: string,\n  numWorkers: number,\n  numTables: number,\n): TransactionPool {\n  const {init} = importSnapshot(snapshot);\n  const tableCopiers = new TransactionPool(\n    lc,\n    Mode.READONLY,\n    init,\n    undefined,\n    numWorkers,\n  );\n  tableCopiers.run(db);\n\n  lc.info?.(`Started ${numWorkers} workers to copy ${numTables} tables`);\n\n  if (parseInt(process.versions.node) < 22) {\n    lc.warn?.(\n      `\\n\\n\\n` +\n        `Older versions of Node have a bug that results in an unresponsive\\n` +\n        `Postgres connection after running certain combinations of COPY commands.\\n` +\n        `If initial sync hangs, run zero-cache with Node v22+. This has the additional\\n` +\n        `benefit of being consistent with the Node version run in the production container image.` +\n        `\\n\\n\\n`,\n    );\n  }\n  return tableCopiers;\n}\n\n// Row returned by `CREATE_REPLICATION_SLOT`\ntype ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  slotName: string,\n): Promise<ReplicationSlot> {\n  const slot = (\n    await session.unsafe<ReplicationSlot[]>(\n      /*sql*/ `CREATE_REPLICATION_SLOT \"${slotName}\" LOGICAL pgoutput`,\n    )\n  )[0];\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\nfunction createLiteTables(\n  tx: Database,\n  tables: PublishedTableSpec[],\n  initialVersion: string,\n) {\n  // TODO: Figure out how to reuse the ChangeProcessor here to avoid\n  //       duplicating the ColumnMetadata logic.\n  const columnMetadata = must(ColumnMetadataStore.getInstance(tx));\n  for (const t of tables) {\n    tx.exec(createLiteTableStatement(mapPostgresToLite(t, initialVersion)));\n    const tableName = liteTableName(t);\n    for (const [colName, colSpec] of Object.entries(t.columns)) {\n      columnMetadata.insert(tableName, colName, colSpec);\n    }\n  }\n}\n\nfunction createLiteIndices(tx: Database, indices: IndexSpec[]) {\n  for (const index of indices) {\n    tx.exec(createLiteIndexStatement(mapPostgresToLiteIndex(index)));\n  }\n}\n\n// Verified empirically that batches of 50 seem to be the sweet spot,\n// similar to the report in https://sqlite.org/forum/forumpost/8878a512d3652655\n//\n// Exported for testing.\nexport const INSERT_BATCH_SIZE = 50;\n\nconst MB = 1024 * 1024;\nconst MAX_BUFFERED_ROWS = 10_000;\nconst BUFFERED_SIZE_THRESHOLD = 8 * MB;\n\nexport type DownloadStatements = {\n  select: string;\n  getTotalRows: string;\n  getTotalBytes: string;\n};\n\nexport function makeDownloadStatements(\n  table: PublishedTableSpec,\n  cols: string[],\n): DownloadStatements {\n  const filterConditions = Object.values(table.publications)\n    .map(({rowFilter}) => rowFilter)\n    .filter(f => !!f); // remove nulls\n  const where =\n    filterConditions.length === 0\n      ? ''\n      : /*sql*/ `WHERE ${filterConditions.join(' OR ')}`;\n  const fromTable = /*sql*/ `FROM ${id(table.schema)}.${id(table.name)} ${where}`;\n  const totalBytes = `(${cols.map(col => `SUM(COALESCE(pg_column_size(${id(col)}), 0))`).join(' + ')})`;\n  const stmts = {\n    select: /*sql*/ `SELECT ${cols.map(id).join(',')} ${fromTable}`,\n    getTotalRows: /*sql*/ `SELECT COUNT(*) AS \"totalRows\" ${fromTable}`,\n    getTotalBytes: /*sql*/ `SELECT ${totalBytes} AS \"totalBytes\" ${fromTable}`,\n  };\n  return stmts;\n}\n\ntype DownloadState = {\n  spec: PublishedTableSpec;\n  status: DownloadStatus;\n};\n\nasync function getInitialDownloadState(\n  lc: LogContext,\n  sql: PostgresDB,\n  spec: PublishedTableSpec,\n): Promise<DownloadState> {\n  const start = performance.now();\n  const table = liteTableName(spec);\n  const columns = Object.keys(spec.columns);\n  const stmts = makeDownloadStatements(spec, columns);\n  const rowsResult = sql\n    .unsafe<{totalRows: bigint}[]>(stmts.getTotalRows)\n    .execute();\n  const bytesResult = sql\n    .unsafe<{totalBytes: bigint}[]>(stmts.getTotalBytes)\n    .execute();\n\n  const state: DownloadState = {\n    spec,\n    status: {\n      table,\n      columns,\n      rows: 0,\n      totalRows: Number((await rowsResult)[0].totalRows),\n      totalBytes: Number((await bytesResult)[0].totalBytes),\n    },\n  };\n  const elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(`Computed initial download state for ${table} (${elapsed} ms)`, {\n    state: state.status,\n  });\n  return state;\n}\n\nasync function copy(\n  lc: LogContext,\n  {spec: table, status}: DownloadState,\n  dbClient: PostgresDB,\n  from: PostgresTransaction,\n  to: Database,\n) {\n  const start = performance.now();\n  let flushTime = 0;\n\n  const tableName = liteTableName(table);\n  const orderedColumns = Object.entries(table.columns);\n\n  const columnNames = orderedColumns.map(([c]) => c);\n  const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n  const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n  // (?,?,?,?,?)\n  const valuesSql =\n    columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n  const insertSql = /*sql*/ `\n    INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n  const insertStmt = to.prepare(insertSql);\n  // INSERT VALUES (?,?,?,?,?),... x INSERT_BATCH_SIZE\n  const insertBatchStmt = to.prepare(\n    insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n  );\n\n  const {select} = makeDownloadStatements(table, columnNames);\n  const valuesPerRow = columnSpecs.length;\n  const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;\n\n  // Preallocate the buffer of values to reduce memory allocation churn.\n  const pendingValues: LiteValueType[] = Array.from({\n    length: MAX_BUFFERED_ROWS * valuesPerRow,\n  });\n  let pendingRows = 0;\n  let pendingSize = 0;\n\n  function flush() {\n    const start = performance.now();\n    const flushedRows = pendingRows;\n    const flushedSize = pendingSize;\n\n    let l = 0;\n    for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n      insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n    }\n    // Insert the remaining rows individually.\n    for (; pendingRows > 0; pendingRows--) {\n      insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n    }\n    for (let i = 0; i < flushedRows; i++) {\n      // Reuse the array and unreference the values to allow GC.\n      // This is faster than allocating a new array every time.\n      pendingValues[i] = undefined as unknown as LiteValueType;\n    }\n    pendingSize = 0;\n    status.rows += flushedRows;\n\n    const elapsed = performance.now() - start;\n    flushTime += elapsed;\n    lc.debug?.(\n      `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n    );\n  }\n\n  lc.info?.(`Starting copy stream of ${tableName}:`, select);\n  const pgParsers = await getTypeParsers(dbClient, {returnJsonAsString: true});\n  const parsers = columnSpecs.map(c => {\n    const pgParse = pgParsers.getTypeParser(c.typeOID);\n    return (val: string) =>\n      liteValue(\n        pgParse(val) as PostgresValueType,\n        c.dataType,\n        JSON_STRINGIFIED,\n      );\n  });\n\n  const tsvParser = new TsvParser();\n  let col = 0;\n\n  await pipeline(\n    await from.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n    new Writable({\n      highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n      write(\n        chunk: Buffer,\n        _encoding: string,\n        callback: (error?: Error) => void,\n      ) {\n        try {\n          for (const text of tsvParser.parse(chunk)) {\n            pendingSize += text === null ? 4 : text.length;\n            pendingValues[pendingRows * valuesPerRow + col] =\n              text === null ? null : parsers[col](text);\n\n            if (++col === parsers.length) {\n              col = 0;\n              if (\n                ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow ||\n                pendingSize >= BUFFERED_SIZE_THRESHOLD\n              ) {\n                flush();\n              }\n            }\n          }\n          callback();\n        } catch (e) {\n          callback(e instanceof Error ? e : new Error(String(e)));\n        }\n      },\n\n      final: (callback: (error?: Error) => void) => {\n        try {\n          flush();\n          callback();\n        } catch (e) {\n          callback(e instanceof Error ? e : new Error(String(e)));\n        }\n      },\n    }),\n  );\n\n  const elapsed = performance.now() - start;\n  lc.info?.(\n    `Finished copying ${status.rows} rows into ${tableName} ` +\n      `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n  );\n  return {rows: status.rows, flushTime};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmEA,eAAsB,YACpB,IACA,OACA,IACA,aACA,aACA,SACA;AACA,KAAI,CAAC,0BAA0B,KAAK,MAAM,MAAM,CAC9C,OAAM,IAAI,MACR,2FACD;CAEH,MAAM,EAAC,kBAAkB,gBAAe;CACxC,MAAM,eAAe,cAAc,MAAM,YAAY,SAAS,GAAG;CACjE,MAAM,MAAM,SAAS,IAAI,YAAY;CACrC,MAAM,qBAAqB,SAAS,IAAI,aAAa;GAClD,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CAAC;CACF,MAAM,WAAW,mBAAmB,MAAM;CAC1C,MAAM,kBAAkB,2BAA2B,sBACjD,GACD,CAAC,QAAQ,IAAI,eAAe;AAC7B,KAAI;AACF,QAAM,oBAAoB,IAAI;EAE9B,MAAM,EAAC,iBAAgB,MAAM,sBAAsB,IAAI,KAAK,MAAM;AAClE,KAAG,OAAO,wCAAwC,aAAa,GAAG;EAElE,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,KAAG,OAAO,kCAAkC,SAAS,GAAG,OAAO;EAE/D,IAAI;AACJ,OAAK,IAAI,QAAQ,OAAQ,QAAQ,MAC/B,KAAI;AACF,UAAO,MAAM,sBAAsB,IAAI,oBAAoB,SAAS;AACpE;WACO,GAAG;AACV,OAAI,SAAS,aAAa,SAAS,eAAe;AAChD,QAAI,EAAE,SAAS,2BAA2B;AAKxC,WAAM,GAAG;AACT,QAAG,OAAO,8CAA8C;AACxD;;AAEF,QAAI,EAAE,SAAS,iCAAiC;KAG9C,MAAM,UAAU,MAAM,GAAqB;;;uCAFpB,0BAA0B,MAAM,CAKb;AAC1C,SAAI,QAAQ,QAAQ;AAClB,SAAG,OACD,uCAAuC,QAAQ,KAAK,EAAC,WAAU,KAAK,IACpE,EACD;AACD;;AAEF,QAAG,QAAQ,oCAAoC,EAAE;;;AAGrD,SAAM;;EAGV,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAAO;EACzD,MAAM,iBAAiB,qBAAqB,IAAI;AAEhD,uBAAqB,IAAI,cAAc,gBAAgB,QAAQ;EAG/D,MAAM,QAAQ,YAAY,KAAK;EAE/B,MAAM,YAAY,MAAM,MACtB,KACA,OAAM,OAAM;AACV,SAAM,GAAG,OAAgB,6BAA6B,SAAS,GAAG;AAClE,UAAO,mBAAmB,IAAI,aAAa;KAE7C,EAAC,MAAM,UAAc,CACtB;AAED,uBAAqB,IAAI,UAAU;EAGnC,MAAM,EAAC,QAAQ,YAAW;EAC1B,MAAM,YAAY,OAAO;AACzB,MAAI,UAAU,KAAK,WAAW,mBAAmB,UAC/C,IAAG,OACD,8CAA8C,iBAAiB,MAC1D,UAAU,gDAChB;EAEH,MAAM,aACJ,UAAU,KAAK,UACX,YACA,KAAK,IAAI,kBAAkB,UAAU;EAE3C,MAAM,WAAW,SAAS,IAAI,aAAa;GACzC,KAAK;GACL,YAAY,GAAE,qBAAqB,4BAA2B;IAC7D,iBAAiB;GACnB,CAAC;EACF,MAAM,UAAU,sBACd,IACA,UACA,UACA,YACA,UACD;AACD,MAAI;AACF,oBAAiB,IAAI,QAAQ,eAAe;GAC5C,MAAM,YAAY,MAAM,QAAQ,IAC9B,OAAO,KAAI,SACT,QAAQ,iBAAiB,IAAI,OAC3B,wBAAwB,IAAI,IAAI,KAAK,CACtC,CACF,CACF;AACD,mBAAgB,QACd,IACA,gBACA,WAAW,UAAU,8BAA8B,kBACnD,YACO,EAAC,gBAAgB,UAAU,KAAK,EAAC,aAAY,OAAO,EAAC,EAC7D;AAEI,iBAAc,OAAO;GAC1B,MAAM,YAAY,MAAM,QAAQ,IAC9B,UAAU,KAAI,UACZ,QAAQ,iBAAiB,IAAI,OAC3B,KAAK,IAAI,OAAO,UAAU,IAAI,GAAG,CAClC,CACF,CACF;AACI,iBAAc,eAAe,IAAI,eAAe;AACrD,WAAQ,SAAS;GAEjB,MAAM,QAAQ,UAAU,QACrB,KAAK,UAAU;IACd,MAAM,IAAI,OAAO,KAAK;IACtB,WAAW,IAAI,YAAY,KAAK;IACjC,GACD;IAAC,MAAM;IAAG,WAAW;IAAE,CACxB;AAED,mBAAgB,QACd,IACA,YACA,YAAY,QAAQ,OAAO,WAC3B,IACD;GACD,MAAM,aAAa,YAAY,KAAK;AACpC,qBAAkB,IAAI,QAAQ;GAC9B,MAAM,QAAQ,YAAY,KAAK,GAAG;AAClC,MAAG,OAAO,oBAAoB,MAAM,QAAQ,EAAE,CAAC,MAAM;AAErD,SAAM,WACJ,KACA,OACA,UACA,gBACA,WACA,QACD;GAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,MAAG,OACD,UAAU,MAAM,KAAK,gBAAgB,CAAC,WAAW,UAAU,aAAa,aAAa,SAAS,IAAI,WACrF,MAAM,UAAU,QAAQ,EAAE,CAAC,WAAW,MAAM,QAAQ,EAAE,CAAC,WAAW,QAAQ,QAAQ,EAAE,CAAC,MACnG;YACO;AAEH,YAAS,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,0BAA0B,EAAE,CAAC;;UAEjE,GAAG;AAIV,KAAG,OAAO,6BAA6B,YAAY,EAAE;AACrD,QAAM,GAAG;;4BAEe,SAAS;MAC/B,OAAM,MAAK,GAAG,OAAO,mCAAmC,YAAY,EAAE,CAAC;AACzE,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;AACtB,QAAM,mBAAmB,KAAK;AAC9B,QAAM,IAAI,KAAK;;;AAInB,eAAe,oBAAoB,KAAiB;CAClD,MAAM,EAAC,UAAU,aACf,MAAM,GAA0C;;;KAIhD;AAEF,KAAI,aAAa,UACf,OAAM,IAAI,MACR,uEAAuE,SAAS,GACjF;AAEH,KAAI,UAAU,KACZ,OAAM,IAAI,MACR,sDAAsD,QAAQ,IAC/D;;AAIL,eAAe,sBACb,IACA,KACA,OACA,WAAW,MACwB;CACnC,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,IAAG,OAAO,oCAAoC,SAAS,GAAG,OAAO;AAEjE,OAAM,kBAAkB,IAAI,KAAK,MAAM;CACvC,MAAM,EAAC,iBAAgB,MAAM,uBAAuB,KAAK,MAAM;AAE/D,KAAI,UAAU;EACZ,IAAI,QAAQ;EACZ,MAAM,0BAA0B,aAAa,QAC3C,MAAK,CAAC,EAAE,WAAW,IAAI,CACxB;EACD,MAAM,SAAS,MAAM,GAAG;4DACgC,IAAI,aAAa,CAAC;QACtE,QAAQ;AACZ,MAAI,OAAO,WAAW,aAAa,OACjC,IAAG,OACD,iCAAiC,aAAa,kBACxC,OAAO,MAAM,CAAC,cACrB;WAED,CAAC,OAAO,IAAI,IAAI,MAAM,aAAa,EAAE,IAAI,IAAI,wBAAwB,CAAC,CAEtE,IAAG,OACD,2BAA2B,MAAM,aAAa,sCAC3B,wBAAwB,cAC5C;MAED,SAAQ;AAEV,MAAI,CAAC,OAAO;AACV,SAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,SAAS,CAAC;AACxD,UAAO,sBAAsB,IAAI,KAAK,OAAO,MAAM;;;AAGvD,QAAO,EAAC,cAAa;;AAGvB,SAAS,sBACP,IACA,IACA,UACA,YACA,WACiB;CACjB,MAAM,EAAC,SAAQ,eAAe,SAAS;CACvC,MAAM,eAAe,IAAI,gBACvB,IACA,UACA,MACA,KAAA,GACA,WACD;AACD,cAAa,IAAI,GAAG;AAEpB,IAAG,OAAO,WAAW,WAAW,mBAAmB,UAAU,SAAS;AAEtE,KAAI,SAAS,QAAQ,SAAS,KAAK,GAAG,GACpC,IAAG,OACD,mUAMD;AAEH,QAAO;;AAcT,eAAsB,sBACpB,IACA,SACA,UAC0B;CAC1B,MAAM,QACJ,MAAM,QAAQ,OACJ,4BAA4B,SAAS,oBAC9C,EACD;AACF,IAAG,OAAO,4BAA4B,YAAY,KAAK;AACvD,QAAO;;AAGT,SAAS,iBACP,IACA,QACA,gBACA;CAGA,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,GAAG,CAAC;AAChE,MAAK,MAAM,KAAK,QAAQ;AACtB,KAAG,KAAK,yBAAyB,kBAAkB,GAAG,eAAe,CAAC,CAAC;EACvE,MAAM,YAAY,cAAc,EAAE;AAClC,OAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,EAAE,QAAQ,CACxD,gBAAe,OAAO,WAAW,SAAS,QAAQ;;;AAKxD,SAAS,kBAAkB,IAAc,SAAsB;AAC7D,MAAK,MAAM,SAAS,QAClB,IAAG,KAAK,yBAAyB,uBAAuB,MAAM,CAAC,CAAC;;AAUpE,IAAM,KAAK,OAAO;AAClB,IAAM,oBAAoB;AAC1B,IAAM,0BAA0B,IAAI;AAQpC,SAAgB,uBACd,OACA,MACoB;CACpB,MAAM,mBAAmB,OAAO,OAAO,MAAM,aAAa,CACvD,KAAK,EAAC,gBAAe,UAAU,CAC/B,QAAO,MAAK,CAAC,CAAC,EAAE;CACnB,MAAM,QACJ,iBAAiB,WAAW,IACxB,KACQ,SAAS,iBAAiB,KAAK,OAAO;CACpD,MAAM,YAAoB,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG;CACxE,MAAM,aAAa,IAAI,KAAK,KAAI,QAAO,+BAA+B,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;AAMnG,QALc;EACZ,QAAgB,UAAU,KAAK,IAAI,GAAG,CAAC,KAAK,IAAI,CAAC,GAAG;EACpD,cAAsB,kCAAkC;EACxD,eAAuB,UAAU,WAAW,mBAAmB;EAChE;;AASH,eAAe,wBACb,IACA,KACA,MACwB;CACxB,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,QAAQ,cAAc,KAAK;CACjC,MAAM,UAAU,OAAO,KAAK,KAAK,QAAQ;CACzC,MAAM,QAAQ,uBAAuB,MAAM,QAAQ;CACnD,MAAM,aAAa,IAChB,OAA8B,MAAM,aAAa,CACjD,SAAS;CACZ,MAAM,cAAc,IACjB,OAA+B,MAAM,cAAc,CACnD,SAAS;CAEZ,MAAM,QAAuB;EAC3B;EACA,QAAQ;GACN;GACA;GACA,MAAM;GACN,WAAW,QAAQ,MAAM,YAAY,GAAG,UAAU;GAClD,YAAY,QAAQ,MAAM,aAAa,GAAG,WAAW;GACtD;EACF;CACD,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,IAAG,OAAO,uCAAuC,MAAM,IAAI,QAAQ,OAAO,EACxE,OAAO,MAAM,QACd,CAAC;AACF,QAAO;;AAGT,eAAe,KACb,IACA,EAAC,MAAM,OAAO,UACd,UACA,MACA,IACA;CACA,MAAM,QAAQ,YAAY,KAAK;CAC/B,IAAI,YAAY;CAEhB,MAAM,YAAY,cAAc,MAAM;CACtC,MAAM,iBAAiB,OAAO,QAAQ,MAAM,QAAQ;CAEpD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,EAAE;CAClD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,UAAU,KAAK;CAC/D,MAAM,mBAAmB,YAAY,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI;CAG9D,MAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,EAAE,CAAC,MAAM;CACzE,MAAM,YAAoB;mBACT,UAAU,KAAK,iBAAiB,WAAW;CAC5D,MAAM,aAAa,GAAG,QAAQ,UAAU;CAExC,MAAM,kBAAkB,GAAG,QACzB,YAAY,IAAI,YAAY,OAAA,GAA6B,CAC1D;CAED,MAAM,EAAC,WAAU,uBAAuB,OAAO,YAAY;CAC3D,MAAM,eAAe,YAAY;CACjC,MAAM,iBAAiB,eAAA;CAGvB,MAAM,gBAAiC,MAAM,KAAK,EAChD,QAAQ,oBAAoB,cAC7B,CAAC;CACF,IAAI,cAAc;CAClB,IAAI,cAAc;CAElB,SAAS,QAAQ;EACf,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,cAAc;EACpB,MAAM,cAAc;EAEpB,IAAI,IAAI;AACR,SAAO,cAAA,IAAiC,eAAA,GACtC,iBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,eAAgB,CAAC;AAGpE,SAAO,cAAc,GAAG,cACtB,YAAW,IAAI,cAAc,MAAM,GAAI,KAAK,aAAc,CAAC;AAE7D,OAAK,IAAI,IAAI,GAAG,IAAI,aAAa,IAG/B,eAAc,KAAK,KAAA;AAErB,gBAAc;AACd,SAAO,QAAQ;EAEf,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,eAAa;AACb,KAAG,QACD,WAAW,YAAY,GAAG,UAAU,SAAS,YAAY,aAAa,QAAQ,QAAQ,EAAE,CAAC,KAC1F;;AAGH,IAAG,OAAO,2BAA2B,UAAU,IAAI,OAAO;CAC1D,MAAM,YAAY,MAAM,eAAe,UAAU,EAAC,oBAAoB,MAAK,CAAC;CAC5E,MAAM,UAAU,YAAY,KAAI,MAAK;EACnC,MAAM,UAAU,UAAU,cAAc,EAAE,QAAQ;AAClD,UAAQ,QACN,UACE,QAAQ,IAAI,EACZ,EAAE,UAAA,IAEH;GACH;CAEF,MAAM,YAAY,IAAI,WAAW;CACjC,IAAI,MAAM;AAEV,OAAM,WACJ,MAAM,KAAK,OAAO,SAAS,OAAO,aAAa,CAAC,UAAU,EAC1D,IAAI,SAAS;EACX,eAAe;EAEf,MACE,OACA,WACA,UACA;AACA,OAAI;AACF,SAAK,MAAM,QAAQ,UAAU,MAAM,MAAM,EAAE;AACzC,oBAAe,SAAS,OAAO,IAAI,KAAK;AACxC,mBAAc,cAAc,eAAe,OACzC,SAAS,OAAO,OAAO,QAAQ,KAAK,KAAK;AAE3C,SAAI,EAAE,QAAQ,QAAQ,QAAQ;AAC5B,YAAM;AACN,UACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,wBAEf,QAAO;;;AAIb,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAI3D,QAAQ,aAAsC;AAC5C,OAAI;AACF,WAAO;AACP,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAG5D,CAAC,CACH;CAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,IAAG,OACD,oBAAoB,OAAO,KAAK,aAAa,UAAU,WAC1C,UAAU,QAAQ,EAAE,CAAC,eAAe,QAAQ,QAAQ,EAAE,CAAC,OACrE;AACD,QAAO;EAAC,MAAM,OAAO;EAAM;EAAU"}
+{"version":3,"file":"initial-sync.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"sourcesContent":["import {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {platform} from 'node:os';\nimport {Writable} from 'node:stream';\nimport {pipeline} from 'node:stream/promises';\nimport postgres from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport type {DownloadStatus} from '../../../../../zero-events/src/status.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {\n  createLiteIndexStatement,\n  createLiteTableStatement,\n} from '../../../db/create.ts';\nimport * as Mode from '../../../db/mode-enum.ts';\nimport {\n  BinaryCopyParser,\n  hasBinaryDecoder,\n  makeBinaryDecoder,\n  textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {\n  mapPostgresToLite,\n  mapPostgresToLiteIndex,\n} from '../../../db/pg-to-lite.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport type {IndexSpec, PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {\n  JSON_STRINGIFIED,\n  liteValue,\n  type LiteValueType,\n} from '../../../types/lite.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport {\n  pgClient,\n  type PostgresDB,\n  type PostgresTransaction,\n  type PostgresValueType,\n} from '../../../types/pg.ts';\nimport {CpuProfiler} from '../../../types/profiler.ts';\nimport type {ShardConfig} from '../../../types/shards.ts';\nimport {ALLOWED_APP_ID_CHARACTERS} from '../../../types/shards.ts';\nimport {id} from '../../../types/sql.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {ColumnMetadataStore} from '../../replicator/schema/column-metadata.ts';\nimport {initReplicationState} from '../../replicator/schema/replication-state.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport {\n  addReplica,\n  dropShard,\n  getInternalShardConfig,\n  newReplicationSlot,\n  replicationSlotExpression,\n  validatePublications,\n} from './schema/shard.ts';\n\nexport type InitialSyncOptions = {\n  tableCopyWorkers: number;\n  profileCopy?: boolean | undefined;\n  textCopy?: boolean | undefined;\n};\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n) {\n  if (!ALLOWED_APP_ID_CHARACTERS.test(shard.appID)) {\n    throw new Error(\n      'The App ID may only consist of lower-case letters, numbers, and the underscore character',\n    );\n  }\n  const {tableCopyWorkers, profileCopy, textCopy = false} = syncOptions;\n  const copyProfiler = profileCopy ? await CpuProfiler.connect() : null;\n  const sql = pgClient(lc, upstreamURI);\n  const replicationSession = pgClient(lc, upstreamURI, {\n    ['fetch_types']: false, // Necessary for the streaming protocol\n    connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n  });\n  const slotName = newReplicationSlot(shard);\n  const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(\n    tx,\n  ).publish(lc, 'Initializing');\n  try {\n    await checkUpstreamConfig(sql);\n\n    const {publications} = await ensurePublishedTables(lc, sql, shard);\n    lc.info?.(`Upstream is setup with publications [${publications}]`);\n\n    const {database, host} = sql.options;\n    lc.info?.(`opening replication session to ${database}@${host}`);\n\n    let slot: ReplicationSlot;\n    for (let first = true; ; first = false) {\n      try {\n        slot = await createReplicationSlot(lc, replicationSession, slotName);\n        break;\n      } catch (e) {\n        if (first && e instanceof postgres.PostgresError) {\n          if (e.code === PG_INSUFFICIENT_PRIVILEGE) {\n            // Some Postgres variants (e.g. Google Cloud SQL) require that\n            // the user have the REPLICATION role in order to create a slot.\n            // Note that this must be done by the upstreamDB connection, and\n            // does not work in the replicationSession itself.\n            await sql`ALTER ROLE current_user WITH REPLICATION`;\n            lc.info?.(`Added the REPLICATION role to database user`);\n            continue;\n          }\n          if (e.code === PG_CONFIGURATION_LIMIT_EXCEEDED) {\n            const slotExpression = replicationSlotExpression(shard);\n\n            const dropped = await sql<{slot: string}[]>`\n              SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n                FROM pg_replication_slots\n                WHERE slot_name LIKE ${slotExpression} AND NOT active`;\n            if (dropped.length) {\n              lc.warn?.(\n                `Dropped inactive replication slots: ${dropped.map(({slot}) => slot)}`,\n                e,\n              );\n              continue;\n            }\n            lc.error?.(`Unable to drop replication slots`, e);\n          }\n        }\n        throw e;\n      }\n    }\n    const {snapshot_name: snapshot, consistent_point: lsn} = slot;\n    const initialVersion = toStateVersionString(lsn);\n\n    initReplicationState(tx, publications, initialVersion, context);\n\n    // Run up to MAX_WORKERS to copy of tables at the replication slot's snapshot.\n    const start = performance.now();\n    // Retrieve the published schema at the consistent_point.\n    const published = await runTx(\n      sql,\n      async tx => {\n        await tx.unsafe(/* sql*/ `SET TRANSACTION SNAPSHOT '${snapshot}'`);\n        return getPublicationInfo(tx, publications);\n      },\n      {mode: Mode.READONLY},\n    );\n    // Note: If this throws, initial-sync is aborted.\n    validatePublications(lc, published);\n\n    // Now that tables have been validated, kick off the copiers.\n    const {tables, indexes} = published;\n    const numTables = tables.length;\n    if (platform() === 'win32' && tableCopyWorkers < numTables) {\n      lc.warn?.(\n        `Increasing the number of copy workers from ${tableCopyWorkers} to ` +\n          `${numTables} to work around a Node/Postgres connection bug`,\n      );\n    }\n    const numWorkers =\n      platform() === 'win32'\n        ? numTables\n        : Math.min(tableCopyWorkers, numTables);\n\n    const copyPool = pgClient(lc, upstreamURI, {\n      max: numWorkers,\n      connection: {['application_name']: 'initial-sync-copy-worker'},\n      ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n    });\n    const copiers = startTableCopyWorkers(\n      lc,\n      copyPool,\n      snapshot,\n      numWorkers,\n      numTables,\n    );\n    try {\n      createLiteTables(tx, tables, initialVersion);\n      const downloads = await Promise.all(\n        tables.map(spec =>\n          copiers.processReadTask((db, lc) =>\n            getInitialDownloadState(lc, db, spec),\n          ),\n        ),\n      );\n      statusPublisher.publish(\n        lc,\n        'Initializing',\n        `Copying ${numTables} upstream tables at version ${initialVersion}`,\n        5000,\n        () => ({downloadStatus: downloads.map(({status}) => status)}),\n      );\n\n      void copyProfiler?.start();\n      const rowCounts = await Promise.all(\n        downloads.map(table =>\n          copiers.processReadTask((db, lc) =>\n            copy(lc, table, copyPool, db, tx, textCopy),\n          ),\n        ),\n      );\n      void copyProfiler?.stopAndDispose(lc, 'initial-copy');\n      copiers.setDone();\n\n      const total = rowCounts.reduce(\n        (acc, curr) => ({\n          rows: acc.rows + curr.rows,\n          flushTime: acc.flushTime + curr.flushTime,\n        }),\n        {rows: 0, flushTime: 0},\n      );\n\n      statusPublisher.publish(\n        lc,\n        'Indexing',\n        `Creating ${indexes.length} indexes`,\n        5000,\n      );\n      const indexStart = performance.now();\n      createLiteIndices(tx, indexes);\n      const index = performance.now() - indexStart;\n      lc.info?.(`Created indexes (${index.toFixed(3)} ms)`);\n\n      await addReplica(\n        sql,\n        shard,\n        slotName,\n        initialVersion,\n        published,\n        context,\n      );\n\n      const elapsed = performance.now() - start;\n      lc.info?.(\n        `Synced ${total.rows.toLocaleString()} rows of ${numTables} tables in ${publications} up to ${lsn} ` +\n          `(flush: ${total.flushTime.toFixed(3)}, index: ${index.toFixed(3)}, total: ${elapsed.toFixed(3)} ms)`,\n      );\n    } finally {\n      // All meaningful errors are handled at the processReadTask() call site.\n      void copyPool.end().catch(e => lc.warn?.(`Error closing copyPool`, e));\n    }\n  } catch (e) {\n    // If initial-sync did not succeed, make a best effort to drop the\n    // orphaned replication slot to avoid running out of slots in\n    // pathological cases that result in repeated failures.\n    lc.warn?.(`dropping replication slot ${slotName}`, e);\n    await sql`\n      SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n        WHERE slot_name = ${slotName};\n    `.catch(e => lc.warn?.(`Unable to drop replication slot ${slotName}`, e));\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n    await replicationSession.end();\n    await sql.end();\n  }\n}\n\nasync function checkUpstreamConfig(sql: PostgresDB) {\n  const {walLevel, version} = (\n    await sql<{walLevel: string; version: number}[]>`\n      SELECT current_setting('wal_level') as \"walLevel\", \n             current_setting('server_version_num') as \"version\";\n  `\n  )[0];\n\n  if (walLevel !== 'logical') {\n    throw new Error(\n      `Postgres must be configured with \"wal_level = logical\" (currently: \"${walLevel})`,\n    );\n  }\n  if (version < 150000) {\n    throw new Error(\n      `Must be running Postgres 15 or higher (currently: \"${version}\")`,\n    );\n  }\n}\n\nasync function ensurePublishedTables(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  validate = true,\n): Promise<{publications: string[]}> {\n  const {database, host} = sql.options;\n  lc.info?.(`Ensuring upstream PUBLICATION on ${database}@${host}`);\n\n  await ensureShardSchema(lc, sql, shard);\n  const {publications} = await getInternalShardConfig(sql, shard);\n\n  if (validate) {\n    let valid = false;\n    const nonInternalPublications = publications.filter(\n      p => !p.startsWith('_'),\n    );\n    const exists = await sql`\n      SELECT pubname FROM pg_publication WHERE pubname IN ${sql(publications)}\n      `.values();\n    if (exists.length !== publications.length) {\n      lc.warn?.(\n        `some configured publications [${publications}] are missing: ` +\n          `[${exists.flat()}]. resyncing`,\n      );\n    } else if (\n      !equals(new Set(shard.publications), new Set(nonInternalPublications))\n    ) {\n      lc.warn?.(\n        `requested publications [${shard.publications}] differ from previous` +\n          `publications [${nonInternalPublications}]. resyncing`,\n      );\n    } else {\n      valid = true;\n    }\n    if (!valid) {\n      await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n      return ensurePublishedTables(lc, sql, shard, false);\n    }\n  }\n  return {publications};\n}\n\nfunction startTableCopyWorkers(\n  lc: LogContext,\n  db: PostgresDB,\n  snapshot: string,\n  numWorkers: number,\n  numTables: number,\n): TransactionPool {\n  const {init} = importSnapshot(snapshot);\n  const tableCopiers = new TransactionPool(\n    lc,\n    Mode.READONLY,\n    init,\n    undefined,\n    numWorkers,\n  );\n  tableCopiers.run(db);\n\n  lc.info?.(`Started ${numWorkers} workers to copy ${numTables} tables`);\n\n  if (parseInt(process.versions.node) < 22) {\n    lc.warn?.(\n      `\\n\\n\\n` +\n        `Older versions of Node have a bug that results in an unresponsive\\n` +\n        `Postgres connection after running certain combinations of COPY commands.\\n` +\n        `If initial sync hangs, run zero-cache with Node v22+. This has the additional\\n` +\n        `benefit of being consistent with the Node version run in the production container image.` +\n        `\\n\\n\\n`,\n    );\n  }\n  return tableCopiers;\n}\n\n// Row returned by `CREATE_REPLICATION_SLOT`\ntype ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  slotName: string,\n): Promise<ReplicationSlot> {\n  const slot = (\n    await session.unsafe<ReplicationSlot[]>(\n      /*sql*/ `CREATE_REPLICATION_SLOT \"${slotName}\" LOGICAL pgoutput`,\n    )\n  )[0];\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\nfunction createLiteTables(\n  tx: Database,\n  tables: PublishedTableSpec[],\n  initialVersion: string,\n) {\n  // TODO: Figure out how to reuse the ChangeProcessor here to avoid\n  //       duplicating the ColumnMetadata logic.\n  const columnMetadata = must(ColumnMetadataStore.getInstance(tx));\n  for (const t of tables) {\n    tx.exec(createLiteTableStatement(mapPostgresToLite(t, initialVersion)));\n    const tableName = liteTableName(t);\n    for (const [colName, colSpec] of Object.entries(t.columns)) {\n      columnMetadata.insert(tableName, colName, colSpec);\n    }\n  }\n}\n\nfunction createLiteIndices(tx: Database, indices: IndexSpec[]) {\n  for (const index of indices) {\n    tx.exec(createLiteIndexStatement(mapPostgresToLiteIndex(index)));\n  }\n}\n\n// Verified empirically that batches of 50 seem to be the sweet spot,\n// similar to the report in https://sqlite.org/forum/forumpost/8878a512d3652655\n//\n// Exported for testing.\nexport const INSERT_BATCH_SIZE = 50;\n\nconst MB = 1024 * 1024;\nconst MAX_BUFFERED_ROWS = 10_000;\nconst BUFFERED_SIZE_THRESHOLD = 8 * MB;\n\nexport type DownloadStatements = {\n  select: string;\n  getTotalRows: string;\n  getTotalBytes: string;\n};\n\nexport function makeDownloadStatements(\n  table: PublishedTableSpec,\n  cols: string[],\n): DownloadStatements {\n  const filterConditions = Object.values(table.publications)\n    .map(({rowFilter}) => rowFilter)\n    .filter(f => !!f); // remove nulls\n  const where =\n    filterConditions.length === 0\n      ? ''\n      : /*sql*/ `WHERE ${filterConditions.join(' OR ')}`;\n  const fromTable = /*sql*/ `FROM ${id(table.schema)}.${id(table.name)} ${where}`;\n  const totalBytes = `(${cols.map(col => `SUM(COALESCE(pg_column_size(${id(col)}), 0))`).join(' + ')})`;\n  const stmts = {\n    select: /*sql*/ `SELECT ${cols.map(id).join(',')} ${fromTable}`,\n    getTotalRows: /*sql*/ `SELECT COUNT(*) AS \"totalRows\" ${fromTable}`,\n    getTotalBytes: /*sql*/ `SELECT ${totalBytes} AS \"totalBytes\" ${fromTable}`,\n  };\n  return stmts;\n}\n\ntype DownloadState = {\n  spec: PublishedTableSpec;\n  status: DownloadStatus;\n};\n\nasync function getInitialDownloadState(\n  lc: LogContext,\n  sql: PostgresDB,\n  spec: PublishedTableSpec,\n): Promise<DownloadState> {\n  const start = performance.now();\n  const table = liteTableName(spec);\n  const columns = Object.keys(spec.columns);\n  const stmts = makeDownloadStatements(spec, columns);\n  const rowsResult = sql\n    .unsafe<{totalRows: bigint}[]>(stmts.getTotalRows)\n    .execute();\n  const bytesResult = sql\n    .unsafe<{totalBytes: bigint}[]>(stmts.getTotalBytes)\n    .execute();\n\n  const state: DownloadState = {\n    spec,\n    status: {\n      table,\n      columns,\n      rows: 0,\n      totalRows: Number((await rowsResult)[0].totalRows),\n      totalBytes: Number((await bytesResult)[0].totalBytes),\n    },\n  };\n  const elapsed = (performance.now() - start).toFixed(3);\n  lc.info?.(`Computed initial download state for ${table} (${elapsed} ms)`, {\n    state: state.status,\n  });\n  return state;\n}\n\nfunction copy(\n  lc: LogContext,\n  {spec: table, status}: DownloadState,\n  dbClient: PostgresDB,\n  from: PostgresTransaction,\n  to: Database,\n  textCopy: boolean,\n) {\n  if (textCopy) {\n    return copyText(lc, table, status, dbClient, from, to);\n  }\n  return copyBinary(lc, table, status, from, to);\n}\n\nasync function copyBinary(\n  lc: LogContext,\n  table: PublishedTableSpec,\n  status: DownloadStatus,\n  from: PostgresTransaction,\n  to: Database,\n) {\n  const start = performance.now();\n  let flushTime = 0;\n\n  const tableName = liteTableName(table);\n  const orderedColumns = Object.entries(table.columns);\n\n  const columnNames = orderedColumns.map(([c]) => c);\n  const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n  const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n  const valuesSql =\n    columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n  const insertSql = /*sql*/ `\n    INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n  const insertStmt = to.prepare(insertSql);\n  const insertBatchStmt = to.prepare(\n    insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n  );\n\n  // Build SELECT with ::text casts for columns without a known binary decoder.\n  const filterConditions = Object.values(table.publications)\n    .map(({rowFilter}) => rowFilter)\n    .filter(f => !!f);\n  const where =\n    filterConditions.length === 0\n      ? ''\n      : /*sql*/ `WHERE ${filterConditions.join(' OR ')}`;\n  const fromTable = /*sql*/ `FROM ${id(table.schema)}.${id(table.name)} ${where}`;\n  const selectColumns = orderedColumns.map(([name, spec]) =>\n    hasBinaryDecoder(spec) ? id(name) : `${id(name)}::text`,\n  );\n  const select = /*sql*/ `SELECT ${selectColumns.join(',')} ${fromTable}`;\n\n  const decoders = orderedColumns.map(([, spec]) =>\n    hasBinaryDecoder(spec) ? makeBinaryDecoder(spec) : textCastDecoder,\n  );\n\n  const valuesPerRow = columnSpecs.length;\n  const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;\n\n  const pendingValues: LiteValueType[] = Array.from({\n    length: MAX_BUFFERED_ROWS * valuesPerRow,\n  });\n  let pendingRows = 0;\n  let pendingSize = 0;\n\n  function flush() {\n    const start = performance.now();\n    const flushedRows = pendingRows;\n    const flushedSize = pendingSize;\n\n    let l = 0;\n    for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n      insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n    }\n    for (; pendingRows > 0; pendingRows--) {\n      insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n    }\n    const flushedValues = flushedRows * valuesPerRow;\n    for (let i = 0; i < flushedValues; i++) {\n      pendingValues[i] = undefined as unknown as LiteValueType;\n    }\n    pendingSize = 0;\n    status.rows += flushedRows;\n\n    const elapsed = performance.now() - start;\n    flushTime += elapsed;\n    lc.debug?.(\n      `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n    );\n  }\n\n  const binaryParser = new BinaryCopyParser();\n  let col = 0;\n\n  lc.info?.(`Starting binary copy stream of ${tableName}:`, select);\n\n  await pipeline(\n    await from\n      .unsafe(`COPY (${select}) TO STDOUT WITH (FORMAT binary)`)\n      .readable(),\n    new Writable({\n      highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n      write(\n        chunk: Buffer,\n        _encoding: string,\n        callback: (error?: Error) => void,\n      ) {\n        try {\n          for (const fieldBuf of binaryParser.parse(chunk)) {\n            pendingSize += fieldBuf === null ? 4 : fieldBuf.length;\n            pendingValues[pendingRows * valuesPerRow + col] =\n              fieldBuf === null ? null : decoders[col](fieldBuf);\n\n            if (++col === decoders.length) {\n              col = 0;\n              if (\n                ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow ||\n                pendingSize >= BUFFERED_SIZE_THRESHOLD\n              ) {\n                flush();\n              }\n            }\n          }\n          callback();\n        } catch (e) {\n          callback(e instanceof Error ? e : new Error(String(e)));\n        }\n      },\n\n      final: (callback: (error?: Error) => void) => {\n        try {\n          flush();\n          callback();\n        } catch (e) {\n          callback(e instanceof Error ? e : new Error(String(e)));\n        }\n      },\n    }),\n  );\n\n  const elapsed = performance.now() - start;\n  lc.info?.(\n    `Finished copying ${status.rows} rows into ${tableName} ` +\n      `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n  );\n  return {rows: status.rows, flushTime};\n}\n\nasync function copyText(\n  lc: LogContext,\n  table: PublishedTableSpec,\n  status: DownloadStatus,\n  dbClient: PostgresDB,\n  from: PostgresTransaction,\n  to: Database,\n) {\n  const start = performance.now();\n  let flushTime = 0;\n\n  const tableName = liteTableName(table);\n  const orderedColumns = Object.entries(table.columns);\n\n  const columnNames = orderedColumns.map(([c]) => c);\n  const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n  const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n  const valuesSql =\n    columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n  const insertSql = /*sql*/ `\n    INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n  const insertStmt = to.prepare(insertSql);\n  const insertBatchStmt = to.prepare(\n    insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n  );\n\n  const {select} = makeDownloadStatements(table, columnNames);\n  const valuesPerRow = columnSpecs.length;\n  const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;\n\n  const pendingValues: LiteValueType[] = Array.from({\n    length: MAX_BUFFERED_ROWS * valuesPerRow,\n  });\n  let pendingRows = 0;\n  let pendingSize = 0;\n\n  function flush() {\n    const start = performance.now();\n    const flushedRows = pendingRows;\n    const flushedSize = pendingSize;\n\n    let l = 0;\n    for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n      insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n    }\n    for (; pendingRows > 0; pendingRows--) {\n      insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n    }\n    const flushedValues = flushedRows * valuesPerRow;\n    for (let i = 0; i < flushedValues; i++) {\n      pendingValues[i] = undefined as unknown as LiteValueType;\n    }\n    pendingSize = 0;\n    status.rows += flushedRows;\n\n    const elapsed = performance.now() - start;\n    flushTime += elapsed;\n    lc.debug?.(\n      `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n    );\n  }\n\n  lc.info?.(`Starting text copy stream of ${tableName}:`, select);\n  const pgParsers = await getTypeParsers(dbClient, {returnJsonAsString: true});\n  const parsers = columnSpecs.map(c => {\n    const pgParse = pgParsers.getTypeParser(c.typeOID);\n    return (val: string) =>\n      liteValue(\n        pgParse(val) as PostgresValueType,\n        c.dataType,\n        JSON_STRINGIFIED,\n      );\n  });\n\n  const tsvParser = new TsvParser();\n  let col = 0;\n\n  await pipeline(\n    await from.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n    new Writable({\n      highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n      write(\n        chunk: Buffer,\n        _encoding: string,\n        callback: (error?: Error) => void,\n      ) {\n        try {\n          for (const text of tsvParser.parse(chunk)) {\n            pendingSize += text === null ? 4 : text.length;\n            pendingValues[pendingRows * valuesPerRow + col] =\n              text === null ? null : parsers[col](text);\n\n            if (++col === parsers.length) {\n              col = 0;\n              if (\n                ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow ||\n                pendingSize >= BUFFERED_SIZE_THRESHOLD\n              ) {\n                flush();\n              }\n            }\n          }\n          callback();\n        } catch (e) {\n          callback(e instanceof Error ? e : new Error(String(e)));\n        }\n      },\n\n      final: (callback: (error?: Error) => void) => {\n        try {\n          flush();\n          callback();\n        } catch (e) {\n          callback(e instanceof Error ? e : new Error(String(e)));\n        }\n      },\n    }),\n  );\n\n  const elapsed = performance.now() - start;\n  lc.info?.(\n    `Finished copying ${status.rows} rows into ${tableName} ` +\n      `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n  );\n  return {rows: status.rows, flushTime};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0EA,eAAsB,YACpB,IACA,OACA,IACA,aACA,aACA,SACA;AACA,KAAI,CAAC,0BAA0B,KAAK,MAAM,MAAM,CAC9C,OAAM,IAAI,MACR,2FACD;CAEH,MAAM,EAAC,kBAAkB,aAAa,WAAW,UAAS;CAC1D,MAAM,eAAe,cAAc,MAAM,YAAY,SAAS,GAAG;CACjE,MAAM,MAAM,SAAS,IAAI,YAAY;CACrC,MAAM,qBAAqB,SAAS,IAAI,aAAa;GAClD,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CAAC;CACF,MAAM,WAAW,mBAAmB,MAAM;CAC1C,MAAM,kBAAkB,2BAA2B,sBACjD,GACD,CAAC,QAAQ,IAAI,eAAe;AAC7B,KAAI;AACF,QAAM,oBAAoB,IAAI;EAE9B,MAAM,EAAC,iBAAgB,MAAM,sBAAsB,IAAI,KAAK,MAAM;AAClE,KAAG,OAAO,wCAAwC,aAAa,GAAG;EAElE,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,KAAG,OAAO,kCAAkC,SAAS,GAAG,OAAO;EAE/D,IAAI;AACJ,OAAK,IAAI,QAAQ,OAAQ,QAAQ,MAC/B,KAAI;AACF,UAAO,MAAM,sBAAsB,IAAI,oBAAoB,SAAS;AACpE;WACO,GAAG;AACV,OAAI,SAAS,aAAa,SAAS,eAAe;AAChD,QAAI,EAAE,SAAS,2BAA2B;AAKxC,WAAM,GAAG;AACT,QAAG,OAAO,8CAA8C;AACxD;;AAEF,QAAI,EAAE,SAAS,iCAAiC;KAG9C,MAAM,UAAU,MAAM,GAAqB;;;uCAFpB,0BAA0B,MAAM,CAKb;AAC1C,SAAI,QAAQ,QAAQ;AAClB,SAAG,OACD,uCAAuC,QAAQ,KAAK,EAAC,WAAU,KAAK,IACpE,EACD;AACD;;AAEF,QAAG,QAAQ,oCAAoC,EAAE;;;AAGrD,SAAM;;EAGV,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAAO;EACzD,MAAM,iBAAiB,qBAAqB,IAAI;AAEhD,uBAAqB,IAAI,cAAc,gBAAgB,QAAQ;EAG/D,MAAM,QAAQ,YAAY,KAAK;EAE/B,MAAM,YAAY,MAAM,MACtB,KACA,OAAM,OAAM;AACV,SAAM,GAAG,OAAgB,6BAA6B,SAAS,GAAG;AAClE,UAAO,mBAAmB,IAAI,aAAa;KAE7C,EAAC,MAAM,UAAc,CACtB;AAED,uBAAqB,IAAI,UAAU;EAGnC,MAAM,EAAC,QAAQ,YAAW;EAC1B,MAAM,YAAY,OAAO;AACzB,MAAI,UAAU,KAAK,WAAW,mBAAmB,UAC/C,IAAG,OACD,8CAA8C,iBAAiB,MAC1D,UAAU,gDAChB;EAEH,MAAM,aACJ,UAAU,KAAK,UACX,YACA,KAAK,IAAI,kBAAkB,UAAU;EAE3C,MAAM,WAAW,SAAS,IAAI,aAAa;GACzC,KAAK;GACL,YAAY,GAAE,qBAAqB,4BAA2B;IAC7D,iBAAiB;GACnB,CAAC;EACF,MAAM,UAAU,sBACd,IACA,UACA,UACA,YACA,UACD;AACD,MAAI;AACF,oBAAiB,IAAI,QAAQ,eAAe;GAC5C,MAAM,YAAY,MAAM,QAAQ,IAC9B,OAAO,KAAI,SACT,QAAQ,iBAAiB,IAAI,OAC3B,wBAAwB,IAAI,IAAI,KAAK,CACtC,CACF,CACF;AACD,mBAAgB,QACd,IACA,gBACA,WAAW,UAAU,8BAA8B,kBACnD,YACO,EAAC,gBAAgB,UAAU,KAAK,EAAC,aAAY,OAAO,EAAC,EAC7D;AAEI,iBAAc,OAAO;GAC1B,MAAM,YAAY,MAAM,QAAQ,IAC9B,UAAU,KAAI,UACZ,QAAQ,iBAAiB,IAAI,OAC3B,KAAK,IAAI,OAAO,UAAU,IAAI,IAAI,SAAS,CAC5C,CACF,CACF;AACI,iBAAc,eAAe,IAAI,eAAe;AACrD,WAAQ,SAAS;GAEjB,MAAM,QAAQ,UAAU,QACrB,KAAK,UAAU;IACd,MAAM,IAAI,OAAO,KAAK;IACtB,WAAW,IAAI,YAAY,KAAK;IACjC,GACD;IAAC,MAAM;IAAG,WAAW;IAAE,CACxB;AAED,mBAAgB,QACd,IACA,YACA,YAAY,QAAQ,OAAO,WAC3B,IACD;GACD,MAAM,aAAa,YAAY,KAAK;AACpC,qBAAkB,IAAI,QAAQ;GAC9B,MAAM,QAAQ,YAAY,KAAK,GAAG;AAClC,MAAG,OAAO,oBAAoB,MAAM,QAAQ,EAAE,CAAC,MAAM;AAErD,SAAM,WACJ,KACA,OACA,UACA,gBACA,WACA,QACD;GAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,MAAG,OACD,UAAU,MAAM,KAAK,gBAAgB,CAAC,WAAW,UAAU,aAAa,aAAa,SAAS,IAAI,WACrF,MAAM,UAAU,QAAQ,EAAE,CAAC,WAAW,MAAM,QAAQ,EAAE,CAAC,WAAW,QAAQ,QAAQ,EAAE,CAAC,MACnG;YACO;AAEH,YAAS,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,0BAA0B,EAAE,CAAC;;UAEjE,GAAG;AAIV,KAAG,OAAO,6BAA6B,YAAY,EAAE;AACrD,QAAM,GAAG;;4BAEe,SAAS;MAC/B,OAAM,MAAK,GAAG,OAAO,mCAAmC,YAAY,EAAE,CAAC;AACzE,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;AACtB,QAAM,mBAAmB,KAAK;AAC9B,QAAM,IAAI,KAAK;;;AAInB,eAAe,oBAAoB,KAAiB;CAClD,MAAM,EAAC,UAAU,aACf,MAAM,GAA0C;;;KAIhD;AAEF,KAAI,aAAa,UACf,OAAM,IAAI,MACR,uEAAuE,SAAS,GACjF;AAEH,KAAI,UAAU,KACZ,OAAM,IAAI,MACR,sDAAsD,QAAQ,IAC/D;;AAIL,eAAe,sBACb,IACA,KACA,OACA,WAAW,MACwB;CACnC,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,IAAG,OAAO,oCAAoC,SAAS,GAAG,OAAO;AAEjE,OAAM,kBAAkB,IAAI,KAAK,MAAM;CACvC,MAAM,EAAC,iBAAgB,MAAM,uBAAuB,KAAK,MAAM;AAE/D,KAAI,UAAU;EACZ,IAAI,QAAQ;EACZ,MAAM,0BAA0B,aAAa,QAC3C,MAAK,CAAC,EAAE,WAAW,IAAI,CACxB;EACD,MAAM,SAAS,MAAM,GAAG;4DACgC,IAAI,aAAa,CAAC;QACtE,QAAQ;AACZ,MAAI,OAAO,WAAW,aAAa,OACjC,IAAG,OACD,iCAAiC,aAAa,kBACxC,OAAO,MAAM,CAAC,cACrB;WAED,CAAC,OAAO,IAAI,IAAI,MAAM,aAAa,EAAE,IAAI,IAAI,wBAAwB,CAAC,CAEtE,IAAG,OACD,2BAA2B,MAAM,aAAa,sCAC3B,wBAAwB,cAC5C;MAED,SAAQ;AAEV,MAAI,CAAC,OAAO;AACV,SAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,SAAS,CAAC;AACxD,UAAO,sBAAsB,IAAI,KAAK,OAAO,MAAM;;;AAGvD,QAAO,EAAC,cAAa;;AAGvB,SAAS,sBACP,IACA,IACA,UACA,YACA,WACiB;CACjB,MAAM,EAAC,SAAQ,eAAe,SAAS;CACvC,MAAM,eAAe,IAAI,gBACvB,IACA,UACA,MACA,KAAA,GACA,WACD;AACD,cAAa,IAAI,GAAG;AAEpB,IAAG,OAAO,WAAW,WAAW,mBAAmB,UAAU,SAAS;AAEtE,KAAI,SAAS,QAAQ,SAAS,KAAK,GAAG,GACpC,IAAG,OACD,mUAMD;AAEH,QAAO;;AAcT,eAAsB,sBACpB,IACA,SACA,UAC0B;CAC1B,MAAM,QACJ,MAAM,QAAQ,OACJ,4BAA4B,SAAS,oBAC9C,EACD;AACF,IAAG,OAAO,4BAA4B,YAAY,KAAK;AACvD,QAAO;;AAGT,SAAS,iBACP,IACA,QACA,gBACA;CAGA,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,GAAG,CAAC;AAChE,MAAK,MAAM,KAAK,QAAQ;AACtB,KAAG,KAAK,yBAAyB,kBAAkB,GAAG,eAAe,CAAC,CAAC;EACvE,MAAM,YAAY,cAAc,EAAE;AAClC,OAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,EAAE,QAAQ,CACxD,gBAAe,OAAO,WAAW,SAAS,QAAQ;;;AAKxD,SAAS,kBAAkB,IAAc,SAAsB;AAC7D,MAAK,MAAM,SAAS,QAClB,IAAG,KAAK,yBAAyB,uBAAuB,MAAM,CAAC,CAAC;;AAUpE,IAAM,KAAK,OAAO;AAClB,IAAM,oBAAoB;AAC1B,IAAM,0BAA0B,IAAI;AAQpC,SAAgB,uBACd,OACA,MACoB;CACpB,MAAM,mBAAmB,OAAO,OAAO,MAAM,aAAa,CACvD,KAAK,EAAC,gBAAe,UAAU,CAC/B,QAAO,MAAK,CAAC,CAAC,EAAE;CACnB,MAAM,QACJ,iBAAiB,WAAW,IACxB,KACQ,SAAS,iBAAiB,KAAK,OAAO;CACpD,MAAM,YAAoB,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG;CACxE,MAAM,aAAa,IAAI,KAAK,KAAI,QAAO,+BAA+B,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;AAMnG,QALc;EACZ,QAAgB,UAAU,KAAK,IAAI,GAAG,CAAC,KAAK,IAAI,CAAC,GAAG;EACpD,cAAsB,kCAAkC;EACxD,eAAuB,UAAU,WAAW,mBAAmB;EAChE;;AASH,eAAe,wBACb,IACA,KACA,MACwB;CACxB,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,QAAQ,cAAc,KAAK;CACjC,MAAM,UAAU,OAAO,KAAK,KAAK,QAAQ;CACzC,MAAM,QAAQ,uBAAuB,MAAM,QAAQ;CACnD,MAAM,aAAa,IAChB,OAA8B,MAAM,aAAa,CACjD,SAAS;CACZ,MAAM,cAAc,IACjB,OAA+B,MAAM,cAAc,CACnD,SAAS;CAEZ,MAAM,QAAuB;EAC3B;EACA,QAAQ;GACN;GACA;GACA,MAAM;GACN,WAAW,QAAQ,MAAM,YAAY,GAAG,UAAU;GAClD,YAAY,QAAQ,MAAM,aAAa,GAAG,WAAW;GACtD;EACF;CACD,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,IAAG,OAAO,uCAAuC,MAAM,IAAI,QAAQ,OAAO,EACxE,OAAO,MAAM,QACd,CAAC;AACF,QAAO;;AAGT,SAAS,KACP,IACA,EAAC,MAAM,OAAO,UACd,UACA,MACA,IACA,UACA;AACA,KAAI,SACF,QAAO,SAAS,IAAI,OAAO,QAAQ,UAAU,MAAM,GAAG;AAExD,QAAO,WAAW,IAAI,OAAO,QAAQ,MAAM,GAAG;;AAGhD,eAAe,WACb,IACA,OACA,QACA,MACA,IACA;CACA,MAAM,QAAQ,YAAY,KAAK;CAC/B,IAAI,YAAY;CAEhB,MAAM,YAAY,cAAc,MAAM;CACtC,MAAM,iBAAiB,OAAO,QAAQ,MAAM,QAAQ;CAEpD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,EAAE;CAClD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,UAAU,KAAK;CAC/D,MAAM,mBAAmB,YAAY,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI;CAE9D,MAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,EAAE,CAAC,MAAM;CACzE,MAAM,YAAoB;mBACT,UAAU,KAAK,iBAAiB,WAAW;CAC5D,MAAM,aAAa,GAAG,QAAQ,UAAU;CACxC,MAAM,kBAAkB,GAAG,QACzB,YAAY,IAAI,YAAY,OAAA,GAA6B,CAC1D;CAGD,MAAM,mBAAmB,OAAO,OAAO,MAAM,aAAa,CACvD,KAAK,EAAC,gBAAe,UAAU,CAC/B,QAAO,MAAK,CAAC,CAAC,EAAE;CACnB,MAAM,QACJ,iBAAiB,WAAW,IACxB,KACQ,SAAS,iBAAiB,KAAK,OAAO;CACpD,MAAM,YAAoB,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG;CAIxE,MAAM,SAAiB,UAHD,eAAe,KAAK,CAAC,MAAM,UAC/C,iBAAiB,KAAK,GAAG,GAAG,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC,QACjD,CAC8C,KAAK,IAAI,CAAC,GAAG;CAE5D,MAAM,WAAW,eAAe,KAAK,GAAG,UACtC,iBAAiB,KAAK,GAAG,kBAAkB,KAAK,GAAG,gBACpD;CAED,MAAM,eAAe,YAAY;CACjC,MAAM,iBAAiB,eAAA;CAEvB,MAAM,gBAAiC,MAAM,KAAK,EAChD,QAAQ,oBAAoB,cAC7B,CAAC;CACF,IAAI,cAAc;CAClB,IAAI,cAAc;CAElB,SAAS,QAAQ;EACf,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,cAAc;EACpB,MAAM,cAAc;EAEpB,IAAI,IAAI;AACR,SAAO,cAAA,IAAiC,eAAA,GACtC,iBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,eAAgB,CAAC;AAEpE,SAAO,cAAc,GAAG,cACtB,YAAW,IAAI,cAAc,MAAM,GAAI,KAAK,aAAc,CAAC;EAE7D,MAAM,gBAAgB,cAAc;AACpC,OAAK,IAAI,IAAI,GAAG,IAAI,eAAe,IACjC,eAAc,KAAK,KAAA;AAErB,gBAAc;AACd,SAAO,QAAQ;EAEf,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,eAAa;AACb,KAAG,QACD,WAAW,YAAY,GAAG,UAAU,SAAS,YAAY,aAAa,QAAQ,QAAQ,EAAE,CAAC,KAC1F;;CAGH,MAAM,eAAe,IAAI,kBAAkB;CAC3C,IAAI,MAAM;AAEV,IAAG,OAAO,kCAAkC,UAAU,IAAI,OAAO;AAEjE,OAAM,WACJ,MAAM,KACH,OAAO,SAAS,OAAO,kCAAkC,CACzD,UAAU,EACb,IAAI,SAAS;EACX,eAAe;EAEf,MACE,OACA,WACA,UACA;AACA,OAAI;AACF,SAAK,MAAM,YAAY,aAAa,MAAM,MAAM,EAAE;AAChD,oBAAe,aAAa,OAAO,IAAI,SAAS;AAChD,mBAAc,cAAc,eAAe,OACzC,aAAa,OAAO,OAAO,SAAS,KAAK,SAAS;AAEpD,SAAI,EAAE,QAAQ,SAAS,QAAQ;AAC7B,YAAM;AACN,UACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,wBAEf,QAAO;;;AAIb,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAI3D,QAAQ,aAAsC;AAC5C,OAAI;AACF,WAAO;AACP,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAG5D,CAAC,CACH;CAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,IAAG,OACD,oBAAoB,OAAO,KAAK,aAAa,UAAU,WAC1C,UAAU,QAAQ,EAAE,CAAC,eAAe,QAAQ,QAAQ,EAAE,CAAC,OACrE;AACD,QAAO;EAAC,MAAM,OAAO;EAAM;EAAU;;AAGvC,eAAe,SACb,IACA,OACA,QACA,UACA,MACA,IACA;CACA,MAAM,QAAQ,YAAY,KAAK;CAC/B,IAAI,YAAY;CAEhB,MAAM,YAAY,cAAc,MAAM;CACtC,MAAM,iBAAiB,OAAO,QAAQ,MAAM,QAAQ;CAEpD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,EAAE;CAClD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,UAAU,KAAK;CAC/D,MAAM,mBAAmB,YAAY,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI;CAE9D,MAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,EAAE,CAAC,MAAM;CACzE,MAAM,YAAoB;mBACT,UAAU,KAAK,iBAAiB,WAAW;CAC5D,MAAM,aAAa,GAAG,QAAQ,UAAU;CACxC,MAAM,kBAAkB,GAAG,QACzB,YAAY,IAAI,YAAY,OAAA,GAA6B,CAC1D;CAED,MAAM,EAAC,WAAU,uBAAuB,OAAO,YAAY;CAC3D,MAAM,eAAe,YAAY;CACjC,MAAM,iBAAiB,eAAA;CAEvB,MAAM,gBAAiC,MAAM,KAAK,EAChD,QAAQ,oBAAoB,cAC7B,CAAC;CACF,IAAI,cAAc;CAClB,IAAI,cAAc;CAElB,SAAS,QAAQ;EACf,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,cAAc;EACpB,MAAM,cAAc;EAEpB,IAAI,IAAI;AACR,SAAO,cAAA,IAAiC,eAAA,GACtC,iBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,eAAgB,CAAC;AAEpE,SAAO,cAAc,GAAG,cACtB,YAAW,IAAI,cAAc,MAAM,GAAI,KAAK,aAAc,CAAC;EAE7D,MAAM,gBAAgB,cAAc;AACpC,OAAK,IAAI,IAAI,GAAG,IAAI,eAAe,IACjC,eAAc,KAAK,KAAA;AAErB,gBAAc;AACd,SAAO,QAAQ;EAEf,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,eAAa;AACb,KAAG,QACD,WAAW,YAAY,GAAG,UAAU,SAAS,YAAY,aAAa,QAAQ,QAAQ,EAAE,CAAC,KAC1F;;AAGH,IAAG,OAAO,gCAAgC,UAAU,IAAI,OAAO;CAC/D,MAAM,YAAY,MAAM,eAAe,UAAU,EAAC,oBAAoB,MAAK,CAAC;CAC5E,MAAM,UAAU,YAAY,KAAI,MAAK;EACnC,MAAM,UAAU,UAAU,cAAc,EAAE,QAAQ;AAClD,UAAQ,QACN,UACE,QAAQ,IAAI,EACZ,EAAE,UAAA,IAEH;GACH;CAEF,MAAM,YAAY,IAAI,WAAW;CACjC,IAAI,MAAM;AAEV,OAAM,WACJ,MAAM,KAAK,OAAO,SAAS,OAAO,aAAa,CAAC,UAAU,EAC1D,IAAI,SAAS;EACX,eAAe;EAEf,MACE,OACA,WACA,UACA;AACA,OAAI;AACF,SAAK,MAAM,QAAQ,UAAU,MAAM,MAAM,EAAE;AACzC,oBAAe,SAAS,OAAO,IAAI,KAAK;AACxC,mBAAc,cAAc,eAAe,OACzC,SAAS,OAAO,OAAO,QAAQ,KAAK,KAAK;AAE3C,SAAI,EAAE,QAAQ,QAAQ,QAAQ;AAC5B,YAAM;AACN,UACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,wBAEf,QAAO;;;AAIb,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAI3D,QAAQ,aAAsC;AAC5C,OAAI;AACF,WAAO;AACP,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAG5D,CAAC,CACH;CAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,IAAG,OACD,oBAAoB,OAAO,KAAK,aAAa,UAAU,WAC1C,UAAU,QAAQ,EAAE,CAAC,eAAe,QAAQ,QAAQ,EAAE,CAAC,OACrE;AACD,QAAO;EAAC,MAAM,OAAO;EAAM;EAAU"}

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js

@@ -2,9 +2,9 @@ import { sleep } from "../../../../../../shared/src/sleep.js";
 import "../../../../types/pg.js";
 import { fromBigInt } from "../lsn.js";
 import { id, lit } from "../../../../types/sql.js";
+import { AutoResetSignal } from "../../../change-streamer/schema/tables.js";
 import { Subscription } from "../../../../types/subscription.js";
 import { pipe } from "../../../../types/streams.js";
-import { AutoResetSignal } from "../../../change-streamer/schema/tables.js";
 import { getTypeParsers } from "../../../../db/pg-type-parser.js";
 import { PgoutputParser } from "./pgoutput-parser.js";
 import postgres from "postgres";

package/out/zero-cache/src/services/change-source/pg/schema/shard.js

@@ -116,7 +116,7 @@ var SHARD_CONFIG_TABLE = "shardConfig";
 function shardSetup(shardConfig, metadataPublication) {
 	const app = id(appSchema(shardConfig));
 	const shard = id(upstreamSchema(shardConfig));
-	const pubs = [...shardConfig.publications].sort();
+	const pubs = shardConfig.publications.toSorted();
 	assert(pubs.includes(metadataPublication), () => `Publications must include ${metadataPublication}`);
 	return `
   CREATE SCHEMA IF NOT EXISTS ${shard};

package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map

@@ -1 +1 @@
-{"version":3,"file":"shard.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {\n  jsonObjectSchema,\n  stringify,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {createEventTriggerStatements} from './ddl.ts';\nimport {\n  getPublicationInfo,\n  publishedSchema,\n  type PublicationInfo,\n  type PublishedSchema,\n} from './published.ts';\nimport {validate} from './validation.ts';\n\n/**\n * PostgreSQL unquoted identifiers must start with a letter or underscore\n * and contain only letters, digits, and underscores.\n */\nconst VALID_PUBLICATION_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;\n\n/**\n * Validates that a publication name is a valid PostgreSQL identifier.\n * This provides defense-in-depth against SQL injection when publication\n * names are used in replication commands.\n */\nexport function validatePublicationName(name: string): void {\n  if (!VALID_PUBLICATION_NAME.test(name)) {\n    throw new Error(\n      `Invalid publication name \"${name}\". Publication names must start with a letter or underscore ` +\n        `and contain only letters, digits, and underscores.`,\n    );\n  }\n  if (name.length > 63) {\n    throw new Error(\n      `Publication name \"${name}\" exceeds PostgreSQL's 63-character identifier limit.`,\n    );\n  }\n}\n\nexport function internalPublicationPrefix({appID}: AppID) {\n  return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n  return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n  const {appID, shardNum} = check(shard);\n  return `${appID}_${shardNum}_`;\n}\n\n/**\n * An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n */\nexport function replicationSlotExpression(shard: ShardID) {\n  // Underscores have a special meaning in LIKE values\n  // so they have to be escaped.\n  return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nexport function newReplicationSlot(shard: ShardID) {\n  return replicationSlotPrefix(shard) + Date.now();\n}\n\nfunction defaultPublicationName(appID: string, shardID: string | number) {\n  return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n  appID: string,\n  shardID: string | number,\n) {\n  return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n  const app = id(appSchema(appID));\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${app};\n\n  CREATE TABLE IF NOT EXISTS ${app}.permissions (\n    \"permissions\" JSONB,\n    \"hash\"        TEXT,\n\n    -- Ensure that there is only a single row in the table.\n    -- Application code can be agnostic to this column, and\n    -- simply invoke UPDATE statements on the version columns.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n  RETURNS TRIGGER AS $$\n  BEGIN\n      NEW.hash = md5(NEW.permissions::text);\n      RETURN NEW;\n  END;\n  $$ LANGUAGE plpgsql;\n\n  CREATE OR REPLACE TRIGGER on_set_permissions \n    BEFORE INSERT OR UPDATE ON ${app}.permissions\n    FOR EACH ROW\n    EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n  INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n  await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"clients\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"lastMutationID\" BIGINT NOT NULL,\n    \"userID\"         TEXT,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\")\n  );`;\n}\n\n/**\n * Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n *\n * The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n */\nexport function getMutationsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"mutations\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"mutationID\"     BIGINT NOT NULL,\n    \"result\"         JSON NOT NULL,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n  );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n  shardConfig: ShardConfig,\n  metadataPublication: string,\n): string {\n  const app = id(appSchema(shardConfig));\n  const shard = id(upstreamSchema(shardConfig));\n\n  const pubs = [...shardConfig.publications].sort();\n  assert(\n    pubs.includes(metadataPublication),\n    () => `Publications must include ${metadataPublication}`,\n  );\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${shard};\n\n  ${getClientsTableDefinition(shard)}\n  ${getMutationsTableDefinition(shard)}\n\n  DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n  CREATE PUBLICATION ${id(metadataPublication)}\n    FOR TABLE ${app}.\"permissions\", TABLE ${shard}.\"clients\", ${shard}.\"mutations\";\n\n  CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n    \"publications\"  TEXT[] NOT NULL,\n    \"ddlDetection\"  BOOL NOT NULL,\n\n    -- Ensure that there is only a single row in the table.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n      \"publications\",\n      \"ddlDetection\" \n    ) VALUES (\n      ARRAY[${literal(pubs)}], \n      false  -- set in SAVEPOINT with triggerSetup() statements\n    );\n\n  CREATE TABLE ${shard}.replicas (\n    \"slot\"               TEXT PRIMARY KEY,\n    \"version\"            TEXT NOT NULL,\n    \"initialSchema\"      JSON NOT NULL,\n    \"initialSyncContext\" JSON,\n    \"subscriberContext\"  JSON\n  );\n  `;\n}\n\nexport function dropShard(appID: string, shardID: string | number): string {\n  const schema = `${appID}_${shardID}`;\n  const metadataPublication = metadataPublicationName(appID, shardID);\n  const defaultPublication = defaultPublicationName(appID, shardID);\n\n  // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n  // so PUBLICATIONs must be dropped explicitly.\n  return /*sql*/ `\n    DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n    DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n    DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n  `;\n}\n\nconst internalShardConfigSchema = v.object({\n  publications: v.array(v.string()),\n  ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaSchema = internalShardConfigSchema.extend({\n  slot: v.string(),\n  version: v.string(),\n  initialSchema: publishedSchema,\n  initialSyncContext: jsonObjectSchema.nullable(),\n  subscriberContext: jsonObjectSchema.nullable(),\n});\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n  const schema = id(upstreamSchema(shard));\n  return (\n    createEventTriggerStatements(shard) +\n    /*sql*/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n  );\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function addReplica(\n  sql: PostgresDB,\n  shard: ShardID,\n  slot: string,\n  replicaVersion: string,\n  {tables, indexes}: PublishedSchema,\n  initialSyncContext: JSONObject,\n) {\n  const schema = upstreamSchema(shard);\n  const synced: PublishedSchema = {tables, indexes};\n  await sql`\n    INSERT INTO ${sql(schema)}.replicas\n      (\"slot\", \"version\", \"initialSchema\", \"initialSyncContext\")\n      VALUES (${slot}, ${replicaVersion}, ${synced}, ${initialSyncContext})`;\n}\n\nexport async function getReplicaAtVersion(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaVersion: string,\n  context?: JSONObject,\n): Promise<Replica | null> {\n  const schema = sql(upstreamSchema(shard));\n  const result = await sql`\n    SELECT * FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n      WHERE version = ${replicaVersion};\n  `;\n  if (result.length === 0) {\n    // log out all the replicas and the joined shardConfig\n    const allReplicas = await sql`\n      SELECT slot, version, \"initialSyncContext\", \"subscriberContext\" \n        FROM ${schema}.replicas`;\n    lc.info?.(\n      `Replica ${replicaVersion} ` +\n        (context ? `(context: ${stringify(context)}) ` : '') +\n        `not found in: ${stringify(allReplicas)}`,\n    );\n    return null;\n  }\n  return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getInternalShardConfig(\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<InternalShardConfig> {\n  const result = await sql`\n    SELECT \"publications\", \"ddlDetection\"\n      FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n  `;\n  assert(\n    result.length === 1,\n    () => `Expected exactly one shardConfig row, got ${result.length}`,\n  );\n  return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/**\n * Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n  lc: LogContext,\n  sql: PostgresTransaction,\n  requested: ShardConfig,\n) {\n  const {publications} = requested;\n  // Validate requested publications.\n  for (const pub of publications) {\n    validatePublicationName(pub);\n    if (pub.startsWith('_')) {\n      throw new Error(\n        `Publication names starting with \"_\" are reserved for internal use.\\n` +\n          `Please use a different name for publication \"${pub}\".`,\n      );\n    }\n  }\n  const allPublications: string[] = [];\n\n  // Setup application publications.\n  if (publications.length) {\n    const results = await sql<{pubname: string}[]>`\n    SELECT pubname from pg_publication WHERE pubname IN ${sql(\n      publications,\n    )}`.values();\n\n    if (results.length !== publications.length) {\n      throw new Error(\n        `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n      );\n    }\n    allPublications.push(...publications);\n  } else {\n    const defaultPublication = defaultPublicationName(\n      requested.appID,\n      requested.shardNum,\n    );\n    await sql`\n      DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n    await sql`\n      CREATE PUBLICATION ${sql(defaultPublication)} \n        FOR TABLES IN SCHEMA public\n        WITH (publish_via_partition_root = true)`;\n    allPublications.push(defaultPublication);\n  }\n\n  const metadataPublication = metadataPublicationName(\n    requested.appID,\n    requested.shardNum,\n  );\n  allPublications.push(metadataPublication);\n\n  const shard = {...requested, publications: allPublications};\n\n  // Setup the global tables and shard tables / publications.\n  await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n  const pubs = await getPublicationInfo(sql, allPublications);\n  await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n  await setupTriggers(lc, sql, shard);\n}\n\nexport async function setupTriggers(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  shard: ShardConfig,\n) {\n  try {\n    await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n  } catch (e) {\n    if (\n      !(\n        e instanceof postgres.PostgresError &&\n        e.code === PG_INSUFFICIENT_PRIVILEGE\n      )\n    ) {\n      throw e;\n    }\n    // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n    lc.warn?.(\n      `Unable to create event triggers for schema change detection:\\n\\n` +\n        `\"${e.hint ?? e.message}\"\\n\\n` +\n        `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n        `requiring the replica to be reset (manually or with --auto-reset).`,\n    );\n  }\n}\n\nexport function validatePublications(\n  lc: LogContext,\n  published: PublicationInfo,\n) {\n  // Verify that all publications export the proper events.\n  published.publications.forEach(pub => {\n    if (\n      !pub.pubinsert ||\n      !pub.pubtruncate ||\n      !pub.pubdelete ||\n      !pub.pubtruncate\n    ) {\n      // TODO: Make APIError?\n      throw new Error(\n        `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n      );\n    }\n  });\n\n  published.tables.forEach(table => validate(lc, table));\n}\n\ntype ReplicaIdentities = {\n  apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n  pubs: PublishedSchema,\n): ReplicaIdentities | undefined {\n  const replicaIdentities: {\n    schema: string;\n    tableName: string;\n    indexName: string;\n  }[] = [];\n  for (const table of pubs.tables) {\n    if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n      // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n      // - UNIQUE\n      // - NOT NULL columns\n      // - not deferrable (i.e. isImmediate)\n      // - not partial (are already filtered out)\n      //\n      // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n      const {schema, name: tableName} = table;\n      for (const {columns, name: indexName} of pubs.indexes.filter(\n        idx =>\n          idx.schema === schema &&\n          idx.tableName === tableName &&\n          idx.unique &&\n          idx.isImmediate,\n      )) {\n        if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n          continue; // Only indexes with all NOT NULL columns are suitable.\n        }\n        replicaIdentities.push({schema, tableName, indexName});\n        break;\n      }\n    }\n  }\n\n  if (replicaIdentities.length === 0) {\n    return undefined;\n  }\n  return {\n    apply: async (lc: LogContext, sql: PostgresDB) => {\n      for (const {schema, tableName, indexName} of replicaIdentities) {\n        lc.info?.(\n          `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n        );\n        await sql`\n        ALTER TABLE ${sql(schema)}.${sql(tableName)} \n          REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n      }\n    },\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA6BA,IAAM,yBAAyB;;;;;;AAO/B,SAAgB,wBAAwB,MAAoB;AAC1D,KAAI,CAAC,uBAAuB,KAAK,KAAK,CACpC,OAAM,IAAI,MACR,6BAA6B,KAAK,gHAEnC;AAEH,KAAI,KAAK,SAAS,GAChB,OAAM,IAAI,MACR,qBAAqB,KAAK,uDAC3B;;AAIL,SAAgB,0BAA0B,EAAC,SAAe;AACxD,QAAO,IAAI,MAAM;;AAGnB,SAAgB,sBAAsB,EAAC,OAAO,YAAoB;AAChE,QAAO,GAAG,MAAM,GAAG;;AAGrB,SAAgB,sBAAsB,OAAgB;CACpD,MAAM,EAAC,OAAO,aAAY,MAAM,MAAM;AACtC,QAAO,GAAG,MAAM,GAAG,SAAS;;;;;;AAO9B,SAAgB,0BAA0B,OAAgB;AAGxD,QAAO,GAAG,sBAAsB,MAAM,CAAC,GAAG,WAAW,KAAK,MAAM;;AAGlE,SAAgB,mBAAmB,OAAgB;AACjD,QAAO,sBAAsB,MAAM,GAAG,KAAK,KAAK;;AAGlD,SAAS,uBAAuB,OAAe,SAA0B;AACvE,QAAO,IAAI,MAAM,UAAU;;AAG7B,SAAgB,wBACd,OACA,SACA;AACA,QAAO,IAAI,MAAM,YAAY;;AAI/B,SAAS,YAAY,OAAsB;CACzC,MAAM,MAAM,GAAG,UAAU,MAAM,CAAC;AAEhC,QAAe;gCACe,IAAI;;+BAEL,IAAI;;;;;;;;;;+BAUJ,IAAI;;;;;;;;;iCASF,IAAI;;uBAEd,IAAI;;gBAEX,IAAI;;;AAIpB,eAAsB,mBAAmB,IAAgB,OAAc;AACrE,OAAM,GAAG,OAAO,YAAY,MAAM,CAAC;;AAGrC,SAAgB,0BAA0B,QAAgB;AACxD,QAAe;iBACA,OAAO;;;;;;;;;;;;;;;;AAiBxB,SAAgB,4BAA4B,QAAgB;AAC1D,QAAe;iBACA,OAAO;;;;;;;;AASxB,IAAa,qBAAqB;AAElC,SAAgB,WACd,aACA,qBACQ;CACR,MAAM,MAAM,GAAG,UAAU,YAAY,CAAC;CACtC,MAAM,QAAQ,GAAG,eAAe,YAAY,CAAC;CAE7C,MAAM,OAAO,CAAC,GAAG,YAAY,aAAa,CAAC,MAAM;AACjD,QACE,KAAK,SAAS,oBAAoB,QAC5B,6BAA6B,sBACpC;AAED,QAAe;gCACe,MAAM;;IAElC,0BAA0B,MAAM,CAAC;IACjC,4BAA4B,MAAM,CAAC;;+BAER,GAAG,oBAAoB,CAAC;uBAChC,GAAG,oBAAoB,CAAC;gBAC/B,IAAI,wBAAwB,MAAM,cAAc,MAAM;;iBAErD,MAAM,IAAI,mBAAmB;;;;;;;;gBAQ9B,MAAM,IAAI,mBAAmB;;;;cAI/B,QAAQ,KAAK,CAAC;;;;iBAIX,MAAM;;;;;;;;;AAUvB,SAAgB,UAAU,OAAe,SAAkC;CACzE,MAAM,SAAS,GAAG,MAAM,GAAG;CAC3B,MAAM,sBAAsB,wBAAwB,OAAO,QAAQ;AAKnE,QAAe;iCACgB,GALJ,uBAAuB,OAAO,QAAQ,CAKZ,CAAC;iCACvB,GAAG,oBAAoB,CAAC;4BAC7B,GAAG,OAAO,CAAC;;;AAIvC,IAAM,4BAA4B,eAAE,OAAO;CACzC,cAAc,eAAE,MAAM,eAAE,QAAQ,CAAC;CACjC,cAAc,eAAE,SAAS;CAC1B,CAAC;AAIF,IAAM,gBAAgB,0BAA0B,OAAO;CACrD,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ;CACnB,eAAe;CACf,oBAAoB,iBAAiB,UAAU;CAC/C,mBAAmB,iBAAiB,UAAU;CAC/C,CAAC;AAOF,SAAS,aAAa,OAA4B;CAChD,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;AACxC,QACE,6BAA6B,MAAM,GAC3B,UAAU,OAAO;;AAK7B,eAAsB,WACpB,KACA,OACA,MACA,gBACA,EAAC,QAAQ,WACT,oBACA;CACA,MAAM,SAAS,eAAe,MAAM;CACpC,MAAM,SAA0B;EAAC;EAAQ;EAAQ;AACjD,OAAM,GAAG;kBACO,IAAI,OAAO,CAAC;;gBAEd,KAAK,IAAI,eAAe,IAAI,OAAO,IAAI,mBAAmB;;AAG1E,eAAsB,oBACpB,IACA,KACA,OACA,gBACA,SACyB;CACzB,MAAM,SAAS,IAAI,eAAe,MAAM,CAAC;CACzC,MAAM,SAAS,MAAM,GAAG;oBACN,OAAO,iBAAiB,OAAO;wBAC3B,eAAe;;AAErC,KAAI,OAAO,WAAW,GAAG;EAEvB,MAAM,cAAc,MAAM,GAAG;;eAElB,OAAO;AAClB,KAAG,OACD,WAAW,eAAe,MACvB,UAAU,aAAa,UAAU,QAAQ,CAAC,MAAM,MACjD,iBAAiB,UAAU,YAAY,GAC1C;AACD,SAAO;;AAET,QAAO,MAAQ,OAAO,IAAI,eAAe,cAAc;;AAGzD,eAAsB,uBACpB,KACA,OAC8B;CAC9B,MAAM,SAAS,MAAM,GAAG;;aAEb,IAAI,eAAe,MAAM,CAAC,CAAC;;AAEtC,QACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,SAC3D;AACD,QAAO,MAAQ,OAAO,IAAI,2BAA2B,cAAc;;;;;;AAOrE,eAAsB,0BACpB,IACA,KACA,WACA;CACA,MAAM,EAAC,iBAAgB;AAEvB,MAAK,MAAM,OAAO,cAAc;AAC9B,0BAAwB,IAAI;AAC5B,MAAI,IAAI,WAAW,IAAI,CACrB,OAAM,IAAI,MACR,oHACkD,IAAI,IACvD;;CAGL,MAAM,kBAA4B,EAAE;AAGpC,KAAI,aAAa,QAAQ;EACvB,MAAM,UAAU,MAAM,GAAwB;0DACQ,IACpD,aACD,GAAG,QAAQ;AAEZ,MAAI,QAAQ,WAAW,aAAa,OAClC,OAAM,IAAI,MACR,gDAAgD,aAAa,aAAa,QAAQ,MAAM,CAAC,GAC1F;AAEH,kBAAgB,KAAK,GAAG,aAAa;QAChC;EACL,MAAM,qBAAqB,uBACzB,UAAU,OACV,UAAU,SACX;AACD,QAAM,GAAG;mCACsB,IAAI,mBAAmB;AACtD,QAAM,GAAG;2BACc,IAAI,mBAAmB,CAAC;;;AAG/C,kBAAgB,KAAK,mBAAmB;;CAG1C,MAAM,sBAAsB,wBAC1B,UAAU,OACV,UAAU,SACX;AACD,iBAAgB,KAAK,oBAAoB;CAEzC,MAAM,QAAQ;EAAC,GAAG;EAAW,cAAc;EAAgB;AAG3D,OAAM,IAAI,OAAO,YAAY,MAAM,GAAG,WAAW,OAAO,oBAAoB,CAAC;AAG7E,OAAM,6CADO,MAAM,mBAAmB,KAAK,gBAAgB,CACH,EAAE,MAAM,IAAI,IAAI;AAExE,OAAM,cAAc,IAAI,KAAK,MAAM;;AAGrC,eAAsB,cACpB,IACA,IACA,OACA;AACA,KAAI;AACF,QAAM,GAAG,WAAU,QAAO,IAAI,OAAO,aAAa,MAAM,CAAC,CAAC;UACnD,GAAG;AACV,MACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,2BAGb,OAAM;AAGR,KAAG,OACD,oEACM,EAAE,QAAQ,EAAE,QAAQ,6IAG3B;;;AAIL,SAAgB,qBACd,IACA,WACA;AAEA,WAAU,aAAa,SAAQ,QAAO;AACpC,MACE,CAAC,IAAI,aACL,CAAC,IAAI,eACL,CAAC,IAAI,aACL,CAAC,IAAI,YAGL,OAAM,IAAI,MACR,eAAe,IAAI,QAAQ,oDAC5B;GAEH;AAEF,WAAU,OAAO,SAAQ,UAAS,SAAS,IAAI,MAAM,CAAC;;AAOxD,SAAgB,6CACd,MAC+B;CAC/B,MAAM,oBAIA,EAAE;AACR,MAAK,MAAM,SAAS,KAAK,OACvB,KAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAA,KAA6B;EAQlE,MAAM,EAAC,QAAQ,MAAM,cAAa;AAClC,OAAK,MAAM,EAAC,SAAS,MAAM,eAAc,KAAK,QAAQ,QACpD,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI,YACP,EAAE;AACD,OAAI,OAAO,KAAK,QAAQ,CAAC,MAAK,QAAO,CAAC,MAAM,QAAQ,KAAK,QAAQ,CAC/D;AAEF,qBAAkB,KAAK;IAAC;IAAQ;IAAW;IAAU,CAAC;AACtD;;;AAKN,KAAI,kBAAkB,WAAW,EAC/B;AAEF,QAAO,EACL,OAAO,OAAO,IAAgB,QAAoB;AAChD,OAAK,MAAM,EAAC,QAAQ,WAAW,eAAc,mBAAmB;AAC9D,MAAG,OACD,YAAY,UAAU,iCAAiC,UAAU,GAClE;AACD,SAAM,GAAG;sBACK,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,CAAC;yCACX,IAAI,UAAU;;IAGpD"}
+{"version":3,"file":"shard.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {\n  jsonObjectSchema,\n  stringify,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {createEventTriggerStatements} from './ddl.ts';\nimport {\n  getPublicationInfo,\n  publishedSchema,\n  type PublicationInfo,\n  type PublishedSchema,\n} from './published.ts';\nimport {validate} from './validation.ts';\n\n/**\n * PostgreSQL unquoted identifiers must start with a letter or underscore\n * and contain only letters, digits, and underscores.\n */\nconst VALID_PUBLICATION_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;\n\n/**\n * Validates that a publication name is a valid PostgreSQL identifier.\n * This provides defense-in-depth against SQL injection when publication\n * names are used in replication commands.\n */\nexport function validatePublicationName(name: string): void {\n  if (!VALID_PUBLICATION_NAME.test(name)) {\n    throw new Error(\n      `Invalid publication name \"${name}\". Publication names must start with a letter or underscore ` +\n        `and contain only letters, digits, and underscores.`,\n    );\n  }\n  if (name.length > 63) {\n    throw new Error(\n      `Publication name \"${name}\" exceeds PostgreSQL's 63-character identifier limit.`,\n    );\n  }\n}\n\nexport function internalPublicationPrefix({appID}: AppID) {\n  return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n  return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n  const {appID, shardNum} = check(shard);\n  return `${appID}_${shardNum}_`;\n}\n\n/**\n * An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n */\nexport function replicationSlotExpression(shard: ShardID) {\n  // Underscores have a special meaning in LIKE values\n  // so they have to be escaped.\n  return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nexport function newReplicationSlot(shard: ShardID) {\n  return replicationSlotPrefix(shard) + Date.now();\n}\n\nfunction defaultPublicationName(appID: string, shardID: string | number) {\n  return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n  appID: string,\n  shardID: string | number,\n) {\n  return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n  const app = id(appSchema(appID));\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${app};\n\n  CREATE TABLE IF NOT EXISTS ${app}.permissions (\n    \"permissions\" JSONB,\n    \"hash\"        TEXT,\n\n    -- Ensure that there is only a single row in the table.\n    -- Application code can be agnostic to this column, and\n    -- simply invoke UPDATE statements on the version columns.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n  RETURNS TRIGGER AS $$\n  BEGIN\n      NEW.hash = md5(NEW.permissions::text);\n      RETURN NEW;\n  END;\n  $$ LANGUAGE plpgsql;\n\n  CREATE OR REPLACE TRIGGER on_set_permissions \n    BEFORE INSERT OR UPDATE ON ${app}.permissions\n    FOR EACH ROW\n    EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n  INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n  await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"clients\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"lastMutationID\" BIGINT NOT NULL,\n    \"userID\"         TEXT,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\")\n  );`;\n}\n\n/**\n * Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n *\n * The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n */\nexport function getMutationsTableDefinition(schema: string) {\n  return /*sql*/ `\n  CREATE TABLE ${schema}.\"mutations\" (\n    \"clientGroupID\"  TEXT NOT NULL,\n    \"clientID\"       TEXT NOT NULL,\n    \"mutationID\"     BIGINT NOT NULL,\n    \"result\"         JSON NOT NULL,\n    PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n  );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n  shardConfig: ShardConfig,\n  metadataPublication: string,\n): string {\n  const app = id(appSchema(shardConfig));\n  const shard = id(upstreamSchema(shardConfig));\n\n  const pubs = shardConfig.publications.toSorted();\n  assert(\n    pubs.includes(metadataPublication),\n    () => `Publications must include ${metadataPublication}`,\n  );\n\n  return /*sql*/ `\n  CREATE SCHEMA IF NOT EXISTS ${shard};\n\n  ${getClientsTableDefinition(shard)}\n  ${getMutationsTableDefinition(shard)}\n\n  DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n  CREATE PUBLICATION ${id(metadataPublication)}\n    FOR TABLE ${app}.\"permissions\", TABLE ${shard}.\"clients\", ${shard}.\"mutations\";\n\n  CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n    \"publications\"  TEXT[] NOT NULL,\n    \"ddlDetection\"  BOOL NOT NULL,\n\n    -- Ensure that there is only a single row in the table.\n    \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n  );\n\n  INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n      \"publications\",\n      \"ddlDetection\" \n    ) VALUES (\n      ARRAY[${literal(pubs)}], \n      false  -- set in SAVEPOINT with triggerSetup() statements\n    );\n\n  CREATE TABLE ${shard}.replicas (\n    \"slot\"               TEXT PRIMARY KEY,\n    \"version\"            TEXT NOT NULL,\n    \"initialSchema\"      JSON NOT NULL,\n    \"initialSyncContext\" JSON,\n    \"subscriberContext\"  JSON\n  );\n  `;\n}\n\nexport function dropShard(appID: string, shardID: string | number): string {\n  const schema = `${appID}_${shardID}`;\n  const metadataPublication = metadataPublicationName(appID, shardID);\n  const defaultPublication = defaultPublicationName(appID, shardID);\n\n  // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n  // so PUBLICATIONs must be dropped explicitly.\n  return /*sql*/ `\n    DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n    DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n    DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n  `;\n}\n\nconst internalShardConfigSchema = v.object({\n  publications: v.array(v.string()),\n  ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaSchema = internalShardConfigSchema.extend({\n  slot: v.string(),\n  version: v.string(),\n  initialSchema: publishedSchema,\n  initialSyncContext: jsonObjectSchema.nullable(),\n  subscriberContext: jsonObjectSchema.nullable(),\n});\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n  const schema = id(upstreamSchema(shard));\n  return (\n    createEventTriggerStatements(shard) +\n    /*sql*/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n  );\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function addReplica(\n  sql: PostgresDB,\n  shard: ShardID,\n  slot: string,\n  replicaVersion: string,\n  {tables, indexes}: PublishedSchema,\n  initialSyncContext: JSONObject,\n) {\n  const schema = upstreamSchema(shard);\n  const synced: PublishedSchema = {tables, indexes};\n  await sql`\n    INSERT INTO ${sql(schema)}.replicas\n      (\"slot\", \"version\", \"initialSchema\", \"initialSyncContext\")\n      VALUES (${slot}, ${replicaVersion}, ${synced}, ${initialSyncContext})`;\n}\n\nexport async function getReplicaAtVersion(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaVersion: string,\n  context?: JSONObject,\n): Promise<Replica | null> {\n  const schema = sql(upstreamSchema(shard));\n  const result = await sql`\n    SELECT * FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n      WHERE version = ${replicaVersion};\n  `;\n  if (result.length === 0) {\n    // log out all the replicas and the joined shardConfig\n    const allReplicas = await sql`\n      SELECT slot, version, \"initialSyncContext\", \"subscriberContext\" \n        FROM ${schema}.replicas`;\n    lc.info?.(\n      `Replica ${replicaVersion} ` +\n        (context ? `(context: ${stringify(context)}) ` : '') +\n        `not found in: ${stringify(allReplicas)}`,\n    );\n    return null;\n  }\n  return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getInternalShardConfig(\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<InternalShardConfig> {\n  const result = await sql`\n    SELECT \"publications\", \"ddlDetection\"\n      FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n  `;\n  assert(\n    result.length === 1,\n    () => `Expected exactly one shardConfig row, got ${result.length}`,\n  );\n  return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/**\n * Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n  lc: LogContext,\n  sql: PostgresTransaction,\n  requested: ShardConfig,\n) {\n  const {publications} = requested;\n  // Validate requested publications.\n  for (const pub of publications) {\n    validatePublicationName(pub);\n    if (pub.startsWith('_')) {\n      throw new Error(\n        `Publication names starting with \"_\" are reserved for internal use.\\n` +\n          `Please use a different name for publication \"${pub}\".`,\n      );\n    }\n  }\n  const allPublications: string[] = [];\n\n  // Setup application publications.\n  if (publications.length) {\n    const results = await sql<{pubname: string}[]>`\n    SELECT pubname from pg_publication WHERE pubname IN ${sql(\n      publications,\n    )}`.values();\n\n    if (results.length !== publications.length) {\n      throw new Error(\n        `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n      );\n    }\n    allPublications.push(...publications);\n  } else {\n    const defaultPublication = defaultPublicationName(\n      requested.appID,\n      requested.shardNum,\n    );\n    await sql`\n      DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n    await sql`\n      CREATE PUBLICATION ${sql(defaultPublication)} \n        FOR TABLES IN SCHEMA public\n        WITH (publish_via_partition_root = true)`;\n    allPublications.push(defaultPublication);\n  }\n\n  const metadataPublication = metadataPublicationName(\n    requested.appID,\n    requested.shardNum,\n  );\n  allPublications.push(metadataPublication);\n\n  const shard = {...requested, publications: allPublications};\n\n  // Setup the global tables and shard tables / publications.\n  await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n  const pubs = await getPublicationInfo(sql, allPublications);\n  await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n  await setupTriggers(lc, sql, shard);\n}\n\nexport async function setupTriggers(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  shard: ShardConfig,\n) {\n  try {\n    await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n  } catch (e) {\n    if (\n      !(\n        e instanceof postgres.PostgresError &&\n        e.code === PG_INSUFFICIENT_PRIVILEGE\n      )\n    ) {\n      throw e;\n    }\n    // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n    lc.warn?.(\n      `Unable to create event triggers for schema change detection:\\n\\n` +\n        `\"${e.hint ?? e.message}\"\\n\\n` +\n        `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n        `requiring the replica to be reset (manually or with --auto-reset).`,\n    );\n  }\n}\n\nexport function validatePublications(\n  lc: LogContext,\n  published: PublicationInfo,\n) {\n  // Verify that all publications export the proper events.\n  published.publications.forEach(pub => {\n    if (\n      !pub.pubinsert ||\n      !pub.pubtruncate ||\n      !pub.pubdelete ||\n      !pub.pubtruncate\n    ) {\n      // TODO: Make APIError?\n      throw new Error(\n        `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n      );\n    }\n  });\n\n  published.tables.forEach(table => validate(lc, table));\n}\n\ntype ReplicaIdentities = {\n  apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n  pubs: PublishedSchema,\n): ReplicaIdentities | undefined {\n  const replicaIdentities: {\n    schema: string;\n    tableName: string;\n    indexName: string;\n  }[] = [];\n  for (const table of pubs.tables) {\n    if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n      // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n      // - UNIQUE\n      // - NOT NULL columns\n      // - not deferrable (i.e. isImmediate)\n      // - not partial (are already filtered out)\n      //\n      // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n      const {schema, name: tableName} = table;\n      for (const {columns, name: indexName} of pubs.indexes.filter(\n        idx =>\n          idx.schema === schema &&\n          idx.tableName === tableName &&\n          idx.unique &&\n          idx.isImmediate,\n      )) {\n        if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n          continue; // Only indexes with all NOT NULL columns are suitable.\n        }\n        replicaIdentities.push({schema, tableName, indexName});\n        break;\n      }\n    }\n  }\n\n  if (replicaIdentities.length === 0) {\n    return undefined;\n  }\n  return {\n    apply: async (lc: LogContext, sql: PostgresDB) => {\n      for (const {schema, tableName, indexName} of replicaIdentities) {\n        lc.info?.(\n          `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n        );\n        await sql`\n        ALTER TABLE ${sql(schema)}.${sql(tableName)} \n          REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n      }\n    },\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA6BA,IAAM,yBAAyB;;;;;;AAO/B,SAAgB,wBAAwB,MAAoB;AAC1D,KAAI,CAAC,uBAAuB,KAAK,KAAK,CACpC,OAAM,IAAI,MACR,6BAA6B,KAAK,gHAEnC;AAEH,KAAI,KAAK,SAAS,GAChB,OAAM,IAAI,MACR,qBAAqB,KAAK,uDAC3B;;AAIL,SAAgB,0BAA0B,EAAC,SAAe;AACxD,QAAO,IAAI,MAAM;;AAGnB,SAAgB,sBAAsB,EAAC,OAAO,YAAoB;AAChE,QAAO,GAAG,MAAM,GAAG;;AAGrB,SAAgB,sBAAsB,OAAgB;CACpD,MAAM,EAAC,OAAO,aAAY,MAAM,MAAM;AACtC,QAAO,GAAG,MAAM,GAAG,SAAS;;;;;;AAO9B,SAAgB,0BAA0B,OAAgB;AAGxD,QAAO,GAAG,sBAAsB,MAAM,CAAC,GAAG,WAAW,KAAK,MAAM;;AAGlE,SAAgB,mBAAmB,OAAgB;AACjD,QAAO,sBAAsB,MAAM,GAAG,KAAK,KAAK;;AAGlD,SAAS,uBAAuB,OAAe,SAA0B;AACvE,QAAO,IAAI,MAAM,UAAU;;AAG7B,SAAgB,wBACd,OACA,SACA;AACA,QAAO,IAAI,MAAM,YAAY;;AAI/B,SAAS,YAAY,OAAsB;CACzC,MAAM,MAAM,GAAG,UAAU,MAAM,CAAC;AAEhC,QAAe;gCACe,IAAI;;+BAEL,IAAI;;;;;;;;;;+BAUJ,IAAI;;;;;;;;;iCASF,IAAI;;uBAEd,IAAI;;gBAEX,IAAI;;;AAIpB,eAAsB,mBAAmB,IAAgB,OAAc;AACrE,OAAM,GAAG,OAAO,YAAY,MAAM,CAAC;;AAGrC,SAAgB,0BAA0B,QAAgB;AACxD,QAAe;iBACA,OAAO;;;;;;;;;;;;;;;;AAiBxB,SAAgB,4BAA4B,QAAgB;AAC1D,QAAe;iBACA,OAAO;;;;;;;;AASxB,IAAa,qBAAqB;AAElC,SAAgB,WACd,aACA,qBACQ;CACR,MAAM,MAAM,GAAG,UAAU,YAAY,CAAC;CACtC,MAAM,QAAQ,GAAG,eAAe,YAAY,CAAC;CAE7C,MAAM,OAAO,YAAY,aAAa,UAAU;AAChD,QACE,KAAK,SAAS,oBAAoB,QAC5B,6BAA6B,sBACpC;AAED,QAAe;gCACe,MAAM;;IAElC,0BAA0B,MAAM,CAAC;IACjC,4BAA4B,MAAM,CAAC;;+BAER,GAAG,oBAAoB,CAAC;uBAChC,GAAG,oBAAoB,CAAC;gBAC/B,IAAI,wBAAwB,MAAM,cAAc,MAAM;;iBAErD,MAAM,IAAI,mBAAmB;;;;;;;;gBAQ9B,MAAM,IAAI,mBAAmB;;;;cAI/B,QAAQ,KAAK,CAAC;;;;iBAIX,MAAM;;;;;;;;;AAUvB,SAAgB,UAAU,OAAe,SAAkC;CACzE,MAAM,SAAS,GAAG,MAAM,GAAG;CAC3B,MAAM,sBAAsB,wBAAwB,OAAO,QAAQ;AAKnE,QAAe;iCACgB,GALJ,uBAAuB,OAAO,QAAQ,CAKZ,CAAC;iCACvB,GAAG,oBAAoB,CAAC;4BAC7B,GAAG,OAAO,CAAC;;;AAIvC,IAAM,4BAA4B,eAAE,OAAO;CACzC,cAAc,eAAE,MAAM,eAAE,QAAQ,CAAC;CACjC,cAAc,eAAE,SAAS;CAC1B,CAAC;AAIF,IAAM,gBAAgB,0BAA0B,OAAO;CACrD,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ;CACnB,eAAe;CACf,oBAAoB,iBAAiB,UAAU;CAC/C,mBAAmB,iBAAiB,UAAU;CAC/C,CAAC;AAOF,SAAS,aAAa,OAA4B;CAChD,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;AACxC,QACE,6BAA6B,MAAM,GAC3B,UAAU,OAAO;;AAK7B,eAAsB,WACpB,KACA,OACA,MACA,gBACA,EAAC,QAAQ,WACT,oBACA;CACA,MAAM,SAAS,eAAe,MAAM;CACpC,MAAM,SAA0B;EAAC;EAAQ;EAAQ;AACjD,OAAM,GAAG;kBACO,IAAI,OAAO,CAAC;;gBAEd,KAAK,IAAI,eAAe,IAAI,OAAO,IAAI,mBAAmB;;AAG1E,eAAsB,oBACpB,IACA,KACA,OACA,gBACA,SACyB;CACzB,MAAM,SAAS,IAAI,eAAe,MAAM,CAAC;CACzC,MAAM,SAAS,MAAM,GAAG;oBACN,OAAO,iBAAiB,OAAO;wBAC3B,eAAe;;AAErC,KAAI,OAAO,WAAW,GAAG;EAEvB,MAAM,cAAc,MAAM,GAAG;;eAElB,OAAO;AAClB,KAAG,OACD,WAAW,eAAe,MACvB,UAAU,aAAa,UAAU,QAAQ,CAAC,MAAM,MACjD,iBAAiB,UAAU,YAAY,GAC1C;AACD,SAAO;;AAET,QAAO,MAAQ,OAAO,IAAI,eAAe,cAAc;;AAGzD,eAAsB,uBACpB,KACA,OAC8B;CAC9B,MAAM,SAAS,MAAM,GAAG;;aAEb,IAAI,eAAe,MAAM,CAAC,CAAC;;AAEtC,QACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,SAC3D;AACD,QAAO,MAAQ,OAAO,IAAI,2BAA2B,cAAc;;;;;;AAOrE,eAAsB,0BACpB,IACA,KACA,WACA;CACA,MAAM,EAAC,iBAAgB;AAEvB,MAAK,MAAM,OAAO,cAAc;AAC9B,0BAAwB,IAAI;AAC5B,MAAI,IAAI,WAAW,IAAI,CACrB,OAAM,IAAI,MACR,oHACkD,IAAI,IACvD;;CAGL,MAAM,kBAA4B,EAAE;AAGpC,KAAI,aAAa,QAAQ;EACvB,MAAM,UAAU,MAAM,GAAwB;0DACQ,IACpD,aACD,GAAG,QAAQ;AAEZ,MAAI,QAAQ,WAAW,aAAa,OAClC,OAAM,IAAI,MACR,gDAAgD,aAAa,aAAa,QAAQ,MAAM,CAAC,GAC1F;AAEH,kBAAgB,KAAK,GAAG,aAAa;QAChC;EACL,MAAM,qBAAqB,uBACzB,UAAU,OACV,UAAU,SACX;AACD,QAAM,GAAG;mCACsB,IAAI,mBAAmB;AACtD,QAAM,GAAG;2BACc,IAAI,mBAAmB,CAAC;;;AAG/C,kBAAgB,KAAK,mBAAmB;;CAG1C,MAAM,sBAAsB,wBAC1B,UAAU,OACV,UAAU,SACX;AACD,iBAAgB,KAAK,oBAAoB;CAEzC,MAAM,QAAQ;EAAC,GAAG;EAAW,cAAc;EAAgB;AAG3D,OAAM,IAAI,OAAO,YAAY,MAAM,GAAG,WAAW,OAAO,oBAAoB,CAAC;AAG7E,OAAM,6CADO,MAAM,mBAAmB,KAAK,gBAAgB,CACH,EAAE,MAAM,IAAI,IAAI;AAExE,OAAM,cAAc,IAAI,KAAK,MAAM;;AAGrC,eAAsB,cACpB,IACA,IACA,OACA;AACA,KAAI;AACF,QAAM,GAAG,WAAU,QAAO,IAAI,OAAO,aAAa,MAAM,CAAC,CAAC;UACnD,GAAG;AACV,MACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,2BAGb,OAAM;AAGR,KAAG,OACD,oEACM,EAAE,QAAQ,EAAE,QAAQ,6IAG3B;;;AAIL,SAAgB,qBACd,IACA,WACA;AAEA,WAAU,aAAa,SAAQ,QAAO;AACpC,MACE,CAAC,IAAI,aACL,CAAC,IAAI,eACL,CAAC,IAAI,aACL,CAAC,IAAI,YAGL,OAAM,IAAI,MACR,eAAe,IAAI,QAAQ,oDAC5B;GAEH;AAEF,WAAU,OAAO,SAAQ,UAAS,SAAS,IAAI,MAAM,CAAC;;AAOxD,SAAgB,6CACd,MAC+B;CAC/B,MAAM,oBAIA,EAAE;AACR,MAAK,MAAM,SAAS,KAAK,OACvB,KAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAA,KAA6B;EAQlE,MAAM,EAAC,QAAQ,MAAM,cAAa;AAClC,OAAK,MAAM,EAAC,SAAS,MAAM,eAAc,KAAK,QAAQ,QACpD,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI,YACP,EAAE;AACD,OAAI,OAAO,KAAK,QAAQ,CAAC,MAAK,QAAO,CAAC,MAAM,QAAQ,KAAK,QAAQ,CAC/D;AAEF,qBAAkB,KAAK;IAAC;IAAQ;IAAW;IAAU,CAAC;AACtD;;;AAKN,KAAI,kBAAkB,WAAW,EAC/B;AAEF,QAAO,EACL,OAAO,OAAO,IAAgB,QAAoB;AAChD,OAAK,MAAM,EAAC,QAAQ,WAAW,eAAc,mBAAmB;AAC9D,MAAG,OACD,YAAY,UAAU,iCAAiC,UAAU,GAClE;AACD,SAAM,GAAG;sBACK,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,CAAC;yCACX,IAAI,UAAU;;IAGpD"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts

@@ -38,7 +38,7 @@ export declare const CHECK_INTERVAL_MS = 60000;
 export declare class BackupMonitor implements Service {
     #private;
     readonly id = "backup-monitor";
-    constructor(lc: LogContext, backupURL: string, metricsEndpoint: string, changeStreamer: ChangeStreamerService, initialCleanupDelayMs: number);
+    constructor(lc: LogContext, replicaFile: string, backupURL: string, metricsEndpoint: string, changeStreamer: ChangeStreamerService, initialCleanupDelayMs: number);
     run(): Promise<void>;
     startSnapshotReservation(taskID: string): Subscription<SnapshotMessage>;
     endReservation(taskID: string, updateCleanupDelay?: boolean): void;

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAEzD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,eAAO,MAAM,iBAAiB,QAAS,CAAC;AAQxC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,aAAc,YAAW,OAAO;;IAC3C,QAAQ,CAAC,EAAE,oBAAoB;gBAe7B,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,qBAAqB,EACrC,qBAAqB,EAAE,MAAM;IAgB/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAYpB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC;IA6BvE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,kBAAkB,UAAO;IAoBxD,QAAQ,CAAC,iCAAiC,sBAWxC;IAwFF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAYtB"}
+{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAEzD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,eAAO,MAAM,iBAAiB,QAAS,CAAC;AAQxC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,aAAc,YAAW,OAAO;;IAC3C,QAAQ,CAAC,EAAE,oBAAoB;gBAiB7B,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,qBAAqB,EACrC,qBAAqB,EAAE,MAAM;IAiB/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAapB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC;IA6BvE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,kBAAkB,UAAO;IAoBxD,QAAQ,CAAC,iCAAiC,sBAWxC;IA0FF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAkDtB"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js

@@ -1,5 +1,7 @@
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { Database } from "../../../../zqlite/src/db.js";
 import { RunningState } from "../running-state.js";
+import { getOrCreateGauge } from "../../observability/metrics.js";
 import { Subscription } from "../../types/subscription.js";
 import parsePrometheusTextFormat from "parse-prometheus-text-format";
 //#region ../zero-cache/src/services/change-streamer/backup-monitor.ts
@@ -39,6 +41,7 @@ var MIN_CLEANUP_DELAY_MS = 3e4;
 var BackupMonitor = class {
 	id = "backup-monitor";
 	#lc;
+	#replicaFile;
 	#backupURL;
 	#metricsEndpoint;
 	#changeStreamer;
@@ -46,10 +49,12 @@ var BackupMonitor = class {
 	#reservations = /* @__PURE__ */ new Map();
 	#watermarks = /* @__PURE__ */ new Map();
 	#lastWatermark = "";
+	#latestBackupTime = null;
 	#cleanupDelayMs;
 	#checkMetricsTimer;
-	constructor(lc, backupURL, metricsEndpoint, changeStreamer, initialCleanupDelayMs) {
+	constructor(lc, replicaFile, backupURL, metricsEndpoint, changeStreamer, initialCleanupDelayMs) {
 		this.#lc = lc.withContext("component", this.id);
+		this.#replicaFile = replicaFile;
 		this.#backupURL = backupURL;
 		this.#metricsEndpoint = metricsEndpoint;
 		this.#changeStreamer = changeStreamer;
@@ -59,6 +64,7 @@ var BackupMonitor = class {
 	run() {
 		this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint} with ${this.#cleanupDelayMs} ms cleanup delay`);
 		this.#checkMetricsTimer = setInterval(this.checkWatermarksAndScheduleCleanup, CHECK_INTERVAL_MS);
+		this.#initBackupLagMetric();
 		return this.#state.stopped();
 	}
 	startSnapshotReservation(taskID) {
@@ -139,7 +145,9 @@ var BackupMonitor = class {
 		for await (const { watermark, name, time } of this.#fetchWatermarks()) if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {
 			this.#lc.info?.(`replicated watermark=${watermark} to ${name} at ${time.toISOString()}.`);
 			this.#watermarks.set(watermark, time);
+			this.#latestBackupTime = time;
 		}
+		return this.#latestBackupTime;
 	}
 	#scheduleCleanup() {
 		if (this.#reservations.size > 0) {
@@ -161,6 +169,28 @@ var BackupMonitor = class {
 		this.#state.stop(this.#lc);
 		return promiseVoid;
 	}
+	#initBackupLagMetric() {
+		getOrCreateGauge("replica", "backup_lag", {
+			description: "Latency from when a change is written to the replica to when it is backed up to litestream. It is expected to create a saw pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.",
+			unit: "millisecond"
+		}).addCallback(async (o) => {
+			const latestBackup = await this.#checkWatermarks();
+			if (!latestBackup) {
+				this.#lc.warn?.(`no backed up watermarks. unable to report replica.backup_lag`);
+				return;
+			}
+			const db = new Database(this.#lc, this.#replicaFile, { readonly: true });
+			try {
+				const { writeTimeMs } = db.prepare(`SELECT writeTimeMs FROM "_zero.replicationState"`).get();
+				const backupLag = Math.max(0, writeTimeMs - latestBackup.getTime());
+				o.observe(backupLag);
+			} catch (e) {
+				this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);
+			} finally {
+				db.close();
+			}
+		});
+	}
 };
 //#endregion
 export { BackupMonitor };

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","names":["#lc","#backupURL","#metricsEndpoint","#changeStreamer","#state","#reservations","#watermarks","#cleanupDelayMs","#checkMetricsTimer","#checkWatermarks","#scheduleCleanup","#fetchWatermarks","#lastWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60_000;\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The BackupMonitor polls the litestream \"/metrics\" endpoint to track the\n * watermark (label) value of the `litestream_replica_progress` gauge and\n * schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n */\nexport class BackupMonitor implements Service {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  #lastWatermark: string = '';\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    // Note: the Subscription must be returned immediately so that the\n    //       websocket can begin sending liveness pings.\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n      }\n    }\n  }\n\n  #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    let maxWatermark = '';\n    for (const [watermark, backupTime] of this.#watermarks.entries()) {\n      if (\n        backupTime.getTime() <= latestCleanupTime &&\n        watermark > maxWatermark\n      ) {\n        maxWatermark = watermark;\n      }\n    }\n    if (maxWatermark.length) {\n      this.#changeStreamer.scheduleCleanup(maxWatermark);\n      for (const watermark of this.#watermarks.keys()) {\n        if (watermark <= maxWatermark) {\n          this.#watermarks.delete(watermark);\n        }\n      }\n      this.#lastWatermark = maxWatermark;\n    }\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;AASA,IAAa,oBAAoB;AACjC,IAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsC7B,IAAa,gBAAb,MAA8C;CAC5C,KAAc;CACd;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,GAAG;CAE3C,gCAAyB,IAAI,KAA0B;CACvD,8BAAuB,IAAI,KAAmB;CAE9C,iBAAyB;CACzB;CACA;CAEA,YACE,IACA,WACA,iBACA,gBACA,uBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,KAAK,GAAG;AAC/C,QAAA,YAAkB;AAClB,QAAA,kBAAwB;AACxB,QAAA,iBAAuB;AACvB,QAAA,iBAAuB,KAAK,IAC1B,uBACA,qBACD;AAED,QAAA,GAAS,OACP,0BAA0B,sBAAsB,4BACjD;;CAGH,MAAqB;AACnB,QAAA,GAAS,OACP,yBAAyB,MAAA,gBAAsB,QAC1C,MAAA,eAAqB,mBAC3B;AACD,QAAA,oBAA0B,YACxB,KAAK,mCACL,kBACD;AACD,SAAO,MAAA,MAAY,SAAS;;CAG9B,yBAAyB,QAA+C;AACtE,QAAA,GAAS,OAAO,oCAAoC,OAAO,YAAY;AAEvE,QAAA,aAAmB,IAAI,OAAO,EAAE,IAAI,QAAQ;EAE5C,MAAM,MAAM,aAAa,OAAwB,EAI/C,eAAe,KAAK,eAAe,QAAQ,MAAM,EAClD,CAAC;AACF,QAAA,aAAmB,IAAI,QAAQ;GAAC,uBAAO,IAAI,MAAM;GAAE;GAAI,CAAC;AAGnD,QAAA,eACF,mBAAmB,CACnB,MAAK,mBAAkB;AACtB,OAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,MAAA;IAAiB,GAAG;IAAe,CAC/D,CAAC;IACF,CACD,OAAM,MAAK;AACV,SAAA,GAAS,OAAO,gCAAgC,EAAE;AAClD,OAAI,KAAK,EAAE;IACX;AACJ,SAAO;;CAGT,eAAe,QAAgB,qBAAqB,MAAM;EACxD,MAAM,MAAM,MAAA,aAAmB,IAAI,OAAO;AAC1C,MAAI,QAAQ,KAAA,EACV;AAEF,QAAA,aAAmB,OAAO,OAAO;EACjC,MAAM,EAAC,OAAO,QAAO;AACrB,MAAI,QAAQ;AAEZ,MAAI,oBAAoB;GACtB,MAAM,WAAW,KAAK,KAAK,GAAG,MAAM,SAAS;AAC7C,SAAA,GAAS,OAAO,2BAA2B,OAAO,MAAM,SAAS,KAAK;AACtE,OAAI,WAAW,MAAA,gBAAsB;AACnC,UAAA,iBAAuB;AACvB,UAAA,GAAS,OAAO,8BAA8B,SAAS,KAAK;;;;CAMlE,oCAA6C,YAAY;AACvD,MAAI;AACF,SAAM,MAAA,iBAAuB;WACtB,GAAG;AACV,SAAA,GAAS,OAAO,8BAA8B,MAAA,mBAAyB,EAAE;;AAE3E,MAAI;AACF,SAAA,iBAAuB;WAChB,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;;;CAIlD,QAAA,kBAIG;EACD,MAAM,kBAAkB,MAAA;EACxB,MAAM,SAAS,MAAA,MAAY;EAC3B,IAAI;AACJ,MAAI;AACF,UAAO,MAAM,MAAM,iBAAiB,EAAC,QAAO,CAAC;WACtC,GAAG;AACV,OAAI,OAAO,QAET;AAIF,SAAA,GAAS,OAAO,8BAA8B,MAAA,mBAAyB,EAAE;AACzE;;AAEF,MAAI,CAAC,KAAK,IAAI;AACZ,SAAA,GAAS,OACP,8BAA8B,MAAA,gBAAsB,IAAI,MAAM,KAAK,MAAM,GAC1E;AACD;;EAGF,MAAM,WAAW,0BAA0B,MAAM,KAAK,MAAM,CAAC;AAC7D,OAAK,MAAM,UAAU,SACnB,KACE,OAAO,SAAS,WAChB,OAAO,SAAS,8BAEhB,MAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,MAAM,GAAG,IAAK;AAEtD,OAAI,UACF,OAAM;IAAC;IAAW;IAAM;IAAK;;;CAOvC,OAAA,kBAAyB;AACvB,aAAW,MAAM,EAAC,WAAW,MAAM,UAAS,MAAA,iBAAuB,CACjE,KAAI,YAAY,MAAA,iBAAuB,CAAC,MAAA,WAAiB,IAAI,UAAU,EAAE;AACvE,SAAA,GAAS,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,aAAa,CAAC,GAC7B;AACD,SAAA,WAAiB,IAAI,WAAW,KAAK;;;CAK3C,mBAAmB;AACjB,MAAI,MAAA,aAAmB,OAAO,GAAG;AAC/B,SAAA,GAAS,OACP,6CAA6C,CAAC,GAAG,MAAA,aAAmB,MAAM,CAAC,GAC5E;AACD;;EAEF,MAAM,oBAAoB,KAAK,KAAK,GAAG,MAAA;EACvC,IAAI,eAAe;AACnB,OAAK,MAAM,CAAC,WAAW,eAAe,MAAA,WAAiB,SAAS,CAC9D,KACE,WAAW,SAAS,IAAI,qBACxB,YAAY,aAEZ,gBAAe;AAGnB,MAAI,aAAa,QAAQ;AACvB,SAAA,eAAqB,gBAAgB,aAAa;AAClD,QAAK,MAAM,aAAa,MAAA,WAAiB,MAAM,CAC7C,KAAI,aAAa,aACf,OAAA,WAAiB,OAAO,UAAU;AAGtC,SAAA,gBAAsB;;;CAI1B,OAAsB;AACpB,gBAAc,MAAA,kBAAwB;AACtC,OAAK,MAAM,EAAC,SAAQ,MAAA,aAAmB,QAAQ,CAK7C,KAAI,QAAQ;AAEd,QAAA,MAAY,KAAK,MAAA,GAAS;AAC1B,SAAO"}
+{"version":3,"file":"backup-monitor.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#changeStreamer","#state","#reservations","#watermarks","#cleanupDelayMs","#checkMetricsTimer","#initBackupLagMetric","#checkWatermarks","#scheduleCleanup","#fetchWatermarks","#lastWatermark","#latestBackupTime"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60_000;\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The BackupMonitor polls the litestream \"/metrics\" endpoint to track the\n * watermark (label) value of the `litestream_replica_progress` gauge and\n * schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n */\nexport class BackupMonitor implements Service {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    this.#initBackupLagMetric();\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    // Note: the Subscription must be returned immediately so that the\n    //       websocket can begin sending liveness pings.\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    let maxWatermark = '';\n    for (const [watermark, backupTime] of this.#watermarks.entries()) {\n      if (\n        backupTime.getTime() <= latestCleanupTime &&\n        watermark > maxWatermark\n      ) {\n        maxWatermark = watermark;\n      }\n    }\n    if (maxWatermark.length) {\n      this.#changeStreamer.scheduleCleanup(maxWatermark);\n      for (const watermark of this.#watermarks.keys()) {\n        if (watermark <= maxWatermark) {\n          this.#watermarks.delete(watermark);\n        }\n      }\n      this.#lastWatermark = maxWatermark;\n    }\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(async o => {\n      // For legacy litestream, we use the watermark metric (and its associated\n      // backup time) exported by litestream metrics to determine the time of\n      // of the backed up watermark. This is technically imprecise--it would be\n      // more correct to use the committed writeTimeMs--but it is good enough\n      // in that it serves the purpose of detecting a non-functioning backup.\n      // With litestream v5, this can be made more precise by querying the\n      // _zero.replicationState row from the backup directly using an LTX-based\n      // database reader.\n      const latestBackup = await this.#checkWatermarks();\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.getTime());\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;AAWA,IAAa,oBAAoB;AACjC,IAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsC7B,IAAa,gBAAb,MAA8C;CAC5C,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,GAAG;CAE3C,gCAAyB,IAAI,KAA0B;CACvD,8BAAuB,IAAI,KAAmB;CAE9C,iBAAyB;CACzB,oBAAiC;CACjC;CACA;CAEA,YACE,IACA,aACA,WACA,iBACA,gBACA,uBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,KAAK,GAAG;AAC/C,QAAA,cAAoB;AACpB,QAAA,YAAkB;AAClB,QAAA,kBAAwB;AACxB,QAAA,iBAAuB;AACvB,QAAA,iBAAuB,KAAK,IAC1B,uBACA,qBACD;AAED,QAAA,GAAS,OACP,0BAA0B,sBAAsB,4BACjD;;CAGH,MAAqB;AACnB,QAAA,GAAS,OACP,yBAAyB,MAAA,gBAAsB,QAC1C,MAAA,eAAqB,mBAC3B;AACD,QAAA,oBAA0B,YACxB,KAAK,mCACL,kBACD;AACD,QAAA,qBAA2B;AAC3B,SAAO,MAAA,MAAY,SAAS;;CAG9B,yBAAyB,QAA+C;AACtE,QAAA,GAAS,OAAO,oCAAoC,OAAO,YAAY;AAEvE,QAAA,aAAmB,IAAI,OAAO,EAAE,IAAI,QAAQ;EAE5C,MAAM,MAAM,aAAa,OAAwB,EAI/C,eAAe,KAAK,eAAe,QAAQ,MAAM,EAClD,CAAC;AACF,QAAA,aAAmB,IAAI,QAAQ;GAAC,uBAAO,IAAI,MAAM;GAAE;GAAI,CAAC;AAGnD,QAAA,eACF,mBAAmB,CACnB,MAAK,mBAAkB;AACtB,OAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,MAAA;IAAiB,GAAG;IAAe,CAC/D,CAAC;IACF,CACD,OAAM,MAAK;AACV,SAAA,GAAS,OAAO,gCAAgC,EAAE;AAClD,OAAI,KAAK,EAAE;IACX;AACJ,SAAO;;CAGT,eAAe,QAAgB,qBAAqB,MAAM;EACxD,MAAM,MAAM,MAAA,aAAmB,IAAI,OAAO;AAC1C,MAAI,QAAQ,KAAA,EACV;AAEF,QAAA,aAAmB,OAAO,OAAO;EACjC,MAAM,EAAC,OAAO,QAAO;AACrB,MAAI,QAAQ;AAEZ,MAAI,oBAAoB;GACtB,MAAM,WAAW,KAAK,KAAK,GAAG,MAAM,SAAS;AAC7C,SAAA,GAAS,OAAO,2BAA2B,OAAO,MAAM,SAAS,KAAK;AACtE,OAAI,WAAW,MAAA,gBAAsB;AACnC,UAAA,iBAAuB;AACvB,UAAA,GAAS,OAAO,8BAA8B,SAAS,KAAK;;;;CAMlE,oCAA6C,YAAY;AACvD,MAAI;AACF,SAAM,MAAA,iBAAuB;WACtB,GAAG;AACV,SAAA,GAAS,OAAO,8BAA8B,MAAA,mBAAyB,EAAE;;AAE3E,MAAI;AACF,SAAA,iBAAuB;WAChB,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;;;CAIlD,QAAA,kBAIG;EACD,MAAM,kBAAkB,MAAA;EACxB,MAAM,SAAS,MAAA,MAAY;EAC3B,IAAI;AACJ,MAAI;AACF,UAAO,MAAM,MAAM,iBAAiB,EAAC,QAAO,CAAC;WACtC,GAAG;AACV,OAAI,OAAO,QAET;AAIF,SAAA,GAAS,OAAO,8BAA8B,MAAA,mBAAyB,EAAE;AACzE;;AAEF,MAAI,CAAC,KAAK,IAAI;AACZ,SAAA,GAAS,OACP,8BAA8B,MAAA,gBAAsB,IAAI,MAAM,KAAK,MAAM,GAC1E;AACD;;EAGF,MAAM,WAAW,0BAA0B,MAAM,KAAK,MAAM,CAAC;AAC7D,OAAK,MAAM,UAAU,SACnB,KACE,OAAO,SAAS,WAChB,OAAO,SAAS,8BAEhB,MAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,MAAM,GAAG,IAAK;AAEtD,OAAI,UACF,OAAM;IAAC;IAAW;IAAM;IAAK;;;CAOvC,OAAA,kBAAyB;AACvB,aAAW,MAAM,EAAC,WAAW,MAAM,UAAS,MAAA,iBAAuB,CACjE,KAAI,YAAY,MAAA,iBAAuB,CAAC,MAAA,WAAiB,IAAI,UAAU,EAAE;AACvE,SAAA,GAAS,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,aAAa,CAAC,GAC7B;AACD,SAAA,WAAiB,IAAI,WAAW,KAAK;AACrC,SAAA,mBAAyB;;AAG7B,SAAO,MAAA;;CAGT,mBAAmB;AACjB,MAAI,MAAA,aAAmB,OAAO,GAAG;AAC/B,SAAA,GAAS,OACP,6CAA6C,CAAC,GAAG,MAAA,aAAmB,MAAM,CAAC,GAC5E;AACD;;EAEF,MAAM,oBAAoB,KAAK,KAAK,GAAG,MAAA;EACvC,IAAI,eAAe;AACnB,OAAK,MAAM,CAAC,WAAW,eAAe,MAAA,WAAiB,SAAS,CAC9D,KACE,WAAW,SAAS,IAAI,qBACxB,YAAY,aAEZ,gBAAe;AAGnB,MAAI,aAAa,QAAQ;AACvB,SAAA,eAAqB,gBAAgB,aAAa;AAClD,QAAK,MAAM,aAAa,MAAA,WAAiB,MAAM,CAC7C,KAAI,aAAa,aACf,OAAA,WAAiB,OAAO,UAAU;AAGtC,SAAA,gBAAsB;;;CAI1B,OAAsB;AACpB,gBAAc,MAAA,kBAAwB;AACtC,OAAK,MAAM,EAAC,SAAQ,MAAA,aAAmB,QAAQ,CAK7C,KAAI,QAAQ;AAEd,QAAA,MAAY,KAAK,MAAA,GAAS;AAC1B,SAAO;;CAGT,uBAAuB;AACrB,mBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;GACP,CAAC,CAAC,YAAY,OAAM,MAAK;GASxB,MAAM,eAAe,MAAM,MAAA,iBAAuB;AAClD,OAAI,CAAC,cAAc;AACjB,UAAA,GAAS,OACP,+DACD;AACD;;GAEF,MAAM,KAAK,IAAI,SAAS,MAAA,IAAU,MAAA,aAAmB,EAAC,UAAU,MAAK,CAAC;AACtE,OAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,mDAAmD,CACnE,KAA4B;IAC/B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,SAAS,CAAC;AACnE,MAAE,QAAQ,UAAU;YACb,GAAG;AACV,UAAA,GAAS,OAAO,6CAA6C,EAAE;aACvD;AACR,OAAG,OAAO;;IAEZ"}

package/out/zero-cache/src/services/change-streamer/broadcast.js

@@ -152,7 +152,7 @@ var Broadcast = class {
 				processed: d.changes,
 				elapsed: d.elapsed
 			})),
-			pending: [...this.#pending].map((sub) => ({
+			pending: Array.from(this.#pending, (sub) => ({
 				id: sub.id,
 				...sub.getStats()
 			}))