npm - @rocicorp/zero - Versions diffs - 1.2.0 → 1.3.0-canary.1 - Mend

@rocicorp/zero 1.2.0 → 1.3.0-canary.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (303) hide show

package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"initial-sync.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"sourcesContent":["import {\n PG_CONFIGURATION_LIMIT_EXCEEDED,\n PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {platform} from 'node:os';\nimport {Writable} from 'node:stream';\nimport {pipeline} from 'node:stream/promises';\nimport postgres from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport type {DownloadStatus} from '../../../../../zero-events/src/status.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {\n createLiteIndexStatement,\n createLiteTableStatement,\n} from '../../../db/create.ts';\nimport * as Mode from '../../../db/mode-enum.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {\n mapPostgresToLite,\n mapPostgresToLiteIndex,\n} from '../../../db/pg-to-lite.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport type {IndexSpec, PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {\n JSON_STRINGIFIED,\n liteValue,\n type LiteValueType,\n} from '../../../types/lite.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport {\n pgClient,\n type PostgresDB,\n type PostgresTransaction,\n type PostgresValueType,\n} from '../../../types/pg.ts';\nimport {CpuProfiler} from '../../../types/profiler.ts';\nimport type {ShardConfig} from '../../../types/shards.ts';\nimport {ALLOWED_APP_ID_CHARACTERS} from '../../../types/shards.ts';\nimport {id} from '../../../types/sql.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {ColumnMetadataStore} from '../../replicator/schema/column-metadata.ts';\nimport {initReplicationState} from '../../replicator/schema/replication-state.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport {\n addReplica,\n dropShard,\n getInternalShardConfig,\n newReplicationSlot,\n replicationSlotExpression,\n validatePublications,\n} from './schema/shard.ts';\n\nexport type InitialSyncOptions = {\n tableCopyWorkers: number;\n profileCopy?: boolean \| undefined;\n};\n\n/** Server context to store with the initial sync metadata for debugging. /\nexport type ServerContext = JSONObject;\n\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n syncOptions: InitialSyncOptions,\n context: ServerContext,\n) {\n if (!ALLOWED_APP_ID_CHARACTERS.test(shard.appID)) {\n throw new Error(\n 'The App ID may only consist of lower-case letters, numbers, and the underscore character',\n );\n }\n const {tableCopyWorkers, profileCopy} = syncOptions;\n const copyProfiler = profileCopy ? await CpuProfiler.connect() : null;\n const sql = pgClient(lc, upstreamURI);\n const replicationSession = pgClient(lc, upstreamURI, {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n });\n const slotName = newReplicationSlot(shard);\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(\n tx,\n ).publish(lc, 'Initializing');\n try {\n await checkUpstreamConfig(sql);\n\n const {publications} = await ensurePublishedTables(lc, sql, shard);\n lc.info?.(`Upstream is setup with publications [${publications}]`);\n\n const {database, host} = sql.options;\n lc.info?.(`opening replication session to ${database}@${host}`);\n\n let slot: ReplicationSlot;\n for (let first = true; ; first = false) {\n try {\n slot = await createReplicationSlot(lc, replicationSession, slotName);\n break;\n } catch (e) {\n if (first && e instanceof postgres.PostgresError) {\n if (e.code === PG_INSUFFICIENT_PRIVILEGE) {\n // Some Postgres variants (e.g. Google Cloud SQL) require that\n // the user have the REPLICATION role in order to create a slot.\n // Note that this must be done by the upstreamDB connection, and\n // does not work in the replicationSession itself.\n await sql`ALTER ROLE current_user WITH REPLICATION`;\n lc.info?.(`Added the REPLICATION role to database user`);\n continue;\n }\n if (e.code === PG_CONFIGURATION_LIMIT_EXCEEDED) {\n const slotExpression = replicationSlotExpression(shard);\n\n const dropped = await sql<{slot: string}[]>`\n SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n FROM pg_replication_slots\n WHERE slot_name LIKE ${slotExpression} AND NOT active`;\n if (dropped.length) {\n lc.warn?.(\n `Dropped inactive replication slots: ${dropped.map(({slot}) => slot)}`,\n e,\n );\n continue;\n }\n lc.error?.(`Unable to drop replication slots`, e);\n }\n }\n throw e;\n }\n }\n const {snapshot_name: snapshot, consistent_point: lsn} = slot;\n const initialVersion = toStateVersionString(lsn);\n\n initReplicationState(tx, publications, initialVersion, context);\n\n // Run up to MAX_WORKERS to copy of tables at the replication slot's snapshot.\n const start = performance.now();\n // Retrieve the published schema at the consistent_point.\n const published = await runTx(\n sql,\n async tx => {\n await tx.unsafe(/ sql/ `SET TRANSACTION SNAPSHOT '${snapshot}'`);\n return getPublicationInfo(tx, publications);\n },\n {mode: Mode.READONLY},\n );\n // Note: If this throws, initial-sync is aborted.\n validatePublications(lc, published);\n\n // Now that tables have been validated, kick off the copiers.\n const {tables, indexes} = published;\n const numTables = tables.length;\n if (platform() === 'win32' && tableCopyWorkers < numTables) {\n lc.warn?.(\n `Increasing the number of copy workers from ${tableCopyWorkers} to ` +\n `${numTables} to work around a Node/Postgres connection bug`,\n );\n }\n const numWorkers =\n platform() === 'win32'\n ? numTables\n : Math.min(tableCopyWorkers, numTables);\n\n const copyPool = pgClient(lc, upstreamURI, {\n max: numWorkers,\n connection: {['application_name']: 'initial-sync-copy-worker'},\n ['max_lifetime']: 120 60, // set a long (2h) limit for COPY streaming\n });\n const copiers = startTableCopyWorkers(\n lc,\n copyPool,\n snapshot,\n numWorkers,\n numTables,\n );\n try {\n createLiteTables(tx, tables, initialVersion);\n const downloads = await Promise.all(\n tables.map(spec =>\n copiers.processReadTask((db, lc) =>\n getInitialDownloadState(lc, db, spec),\n ),\n ),\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying ${numTables} upstream tables at version ${initialVersion}`,\n 5000,\n () => ({downloadStatus: downloads.map(({status}) => status)}),\n );\n\n void copyProfiler?.start();\n const rowCounts = await Promise.all(\n downloads.map(table =>\n copiers.processReadTask((db, lc) =>\n copy(lc, table, copyPool, db, tx),\n ),\n ),\n );\n void copyProfiler?.stopAndDispose(lc, 'initial-copy');\n copiers.setDone();\n\n const total = rowCounts.reduce(\n (acc, curr) => ({\n rows: acc.rows + curr.rows,\n flushTime: acc.flushTime + curr.flushTime,\n }),\n {rows: 0, flushTime: 0},\n );\n\n statusPublisher.publish(\n lc,\n 'Indexing',\n `Creating ${indexes.length} indexes`,\n 5000,\n );\n const indexStart = performance.now();\n createLiteIndices(tx, indexes);\n const index = performance.now() - indexStart;\n lc.info?.(`Created indexes (${index.toFixed(3)} ms)`);\n\n await addReplica(\n sql,\n shard,\n slotName,\n initialVersion,\n published,\n context,\n );\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Synced ${total.rows.toLocaleString()} rows of ${numTables} tables in ${publications} up to ${lsn} ` +\n `(flush: ${total.flushTime.toFixed(3)}, index: ${index.toFixed(3)}, total: ${elapsed.toFixed(3)} ms)`,\n );\n } finally {\n // All meaningful errors are handled at the processReadTask() call site.\n void copyPool.end().catch(e => lc.warn?.(`Error closing copyPool`, e));\n }\n } catch (e) {\n // If initial-sync did not succeed, make a best effort to drop the\n // orphaned replication slot to avoid running out of slots in\n // pathological cases that result in repeated failures.\n lc.warn?.(`dropping replication slot ${slotName}`, e);\n await sql`\n SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n WHERE slot_name = ${slotName};\n `.catch(e => lc.warn?.(`Unable to drop replication slot ${slotName}`, e));\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n await replicationSession.end();\n await sql.end();\n }\n}\n\nasync function checkUpstreamConfig(sql: PostgresDB) {\n const {walLevel, version} = (\n await sql<{walLevel: string; version: number}[]>`\n SELECT current_setting('wal_level') as \"walLevel\", \n current_setting('server_version_num') as \"version\";\n `\n )[0];\n\n if (walLevel !== 'logical') {\n throw new Error(\n `Postgres must be configured with \"wal_level = logical\" (currently: \"${walLevel})`,\n );\n }\n if (version < 150000) {\n throw new Error(\n `Must be running Postgres 15 or higher (currently: \"${version}\")`,\n );\n }\n}\n\nasync function ensurePublishedTables(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardConfig,\n validate = true,\n): Promise<{publications: string[]}> {\n const {database, host} = sql.options;\n lc.info?.(`Ensuring upstream PUBLICATION on ${database}@${host}`);\n\n await ensureShardSchema(lc, sql, shard);\n const {publications} = await getInternalShardConfig(sql, shard);\n\n if (validate) {\n let valid = false;\n const nonInternalPublications = publications.filter(\n p => !p.startsWith('_'),\n );\n const exists = await sql`\n SELECT pubname FROM pg_publication WHERE pubname IN ${sql(publications)}\n `.values();\n if (exists.length !== publications.length) {\n lc.warn?.(\n `some configured publications [${publications}] are missing: ` +\n `[${exists.flat()}]. resyncing`,\n );\n } else if (\n !equals(new Set(shard.publications), new Set(nonInternalPublications))\n ) {\n lc.warn?.(\n `requested publications [${shard.publications}] differ from previous` +\n `publications [${nonInternalPublications}]. resyncing`,\n );\n } else {\n valid = true;\n }\n if (!valid) {\n await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n return ensurePublishedTables(lc, sql, shard, false);\n }\n }\n return {publications};\n}\n\nfunction startTableCopyWorkers(\n lc: LogContext,\n db: PostgresDB,\n snapshot: string,\n numWorkers: number,\n numTables: number,\n): TransactionPool {\n const {init} = importSnapshot(snapshot);\n const tableCopiers = new TransactionPool(\n lc,\n Mode.READONLY,\n init,\n undefined,\n numWorkers,\n );\n tableCopiers.run(db);\n\n lc.info?.(`Started ${numWorkers} workers to copy ${numTables} tables`);\n\n if (parseInt(process.versions.node) < 22) {\n lc.warn?.(\n `\\n\\n\\n` +\n `Older versions of Node have a bug that results in an unresponsive\\n` +\n `Postgres connection after running certain combinations of COPY commands.\\n` +\n `If initial sync hangs, run zero-cache with Node v22+. This has the additional\\n` +\n `benefit of being consistent with the Node version run in the production container image.` +\n `\\n\\n\\n`,\n );\n }\n return tableCopiers;\n}\n\n// Row returned by `CREATE_REPLICATION_SLOT`\ntype ReplicationSlot = {\n slot_name: string;\n consistent_point: string;\n snapshot_name: string;\n output_plugin: string;\n};\n\n// Note: The replication connection does not support the extended query protocol,\n// so all commands must be sent using sql.unsafe(). This is technically safe\n// because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n lc: LogContext,\n session: postgres.Sql,\n slotName: string,\n): Promise<ReplicationSlot> {\n const slot = (\n await session.unsafe<ReplicationSlot[]>(\n /sql/ `CREATE_REPLICATION_SLOT \"${slotName}\" LOGICAL pgoutput`,\n )\n )[0];\n lc.info?.(`Created replication slot ${slotName}`, slot);\n return slot;\n}\n\nfunction createLiteTables(\n tx: Database,\n tables: PublishedTableSpec[],\n initialVersion: string,\n) {\n // TODO: Figure out how to reuse the ChangeProcessor here to avoid\n // duplicating the ColumnMetadata logic.\n const columnMetadata = must(ColumnMetadataStore.getInstance(tx));\n for (const t of tables) {\n tx.exec(createLiteTableStatement(mapPostgresToLite(t, initialVersion)));\n const tableName = liteTableName(t);\n for (const [colName, colSpec] of Object.entries(t.columns)) {\n columnMetadata.insert(tableName, colName, colSpec);\n }\n }\n}\n\nfunction createLiteIndices(tx: Database, indices: IndexSpec[]) {\n for (const index of indices) {\n tx.exec(createLiteIndexStatement(mapPostgresToLiteIndex(index)));\n }\n}\n\n// Verified empirically that batches of 50 seem to be the sweet spot,\n// similar to the report in https://sqlite.org/forum/forumpost/8878a512d3652655\n//\n// Exported for testing.\nexport const INSERT_BATCH_SIZE = 50;\n\nconst MB = 1024 * 1024;\nconst MAX_BUFFERED_ROWS = 10_000;\nconst BUFFERED_SIZE_THRESHOLD = 8 * MB;\n\nexport type DownloadStatements = {\n select: string;\n getTotalRows: string;\n getTotalBytes: string;\n};\n\nexport function makeDownloadStatements(\n table: PublishedTableSpec,\n cols: string[],\n): DownloadStatements {\n const filterConditions = Object.values(table.publications)\n .map(({rowFilter}) => rowFilter)\n .filter(f => !!f); // remove nulls\n const where =\n filterConditions.length === 0\n ? ''\n : /sql/ `WHERE ${filterConditions.join(' OR ')}`;\n const fromTable = /sql/ `FROM ${id(table.schema)}.${id(table.name)} ${where}`;\n const totalBytes = `(${cols.map(col => `SUM(COALESCE(pg_column_size(${id(col)}), 0))`).join(' + ')})`;\n const stmts = {\n select: /sql/ `SELECT ${cols.map(id).join(',')} ${fromTable}`,\n getTotalRows: /sql/ `SELECT COUNT() AS \"totalRows\" ${fromTable}`,\n getTotalBytes: /sql/ `SELECT ${totalBytes} AS \"totalBytes\" ${fromTable}`,\n };\n return stmts;\n}\n\ntype DownloadState = {\n spec: PublishedTableSpec;\n status: DownloadStatus;\n};\n\nasync function getInitialDownloadState(\n lc: LogContext,\n sql: PostgresDB,\n spec: PublishedTableSpec,\n): Promise<DownloadState> {\n const start = performance.now();\n const table = liteTableName(spec);\n const columns = Object.keys(spec.columns);\n const stmts = makeDownloadStatements(spec, columns);\n const rowsResult = sql\n .unsafe<{totalRows: bigint}[]>(stmts.getTotalRows)\n .execute();\n const bytesResult = sql\n .unsafe<{totalBytes: bigint}[]>(stmts.getTotalBytes)\n .execute();\n\n const state: DownloadState = {\n spec,\n status: {\n table,\n columns,\n rows: 0,\n totalRows: Number((await rowsResult)[0].totalRows),\n totalBytes: Number((await bytesResult)[0].totalBytes),\n },\n };\n const elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(`Computed initial download state for ${table} (${elapsed} ms)`, {\n state: state.status,\n });\n return state;\n}\n\nasync function copy(\n lc: LogContext,\n {spec: table, status}: DownloadState,\n dbClient: PostgresDB,\n from: PostgresTransaction,\n to: Database,\n) {\n const start = performance.now();\n let flushTime = 0;\n\n const tableName = liteTableName(table);\n const orderedColumns = Object.entries(table.columns);\n\n const columnNames = orderedColumns.map(([c]) => c);\n const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n // (?,?,?,?,?)\n const valuesSql =\n columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n const insertSql = /sql/ `\n INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n const insertStmt = to.prepare(insertSql);\n // INSERT VALUES (?,?,?,?,?),... x INSERT_BATCH_SIZE\n const insertBatchStmt = to.prepare(\n insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n );\n\n const {select} = makeDownloadStatements(table, columnNames);\n const valuesPerRow = columnSpecs.length;\n const valuesPerBatch = valuesPerRow INSERT_BATCH_SIZE;\n\n // Preallocate the buffer of values to reduce memory allocation churn.\n const pendingValues: LiteValueType[] = Array.from({\n length: MAX_BUFFERED_ROWS * valuesPerRow,\n });\n let pendingRows = 0;\n let pendingSize = 0;\n\n function flush() {\n const start = performance.now();\n const flushedRows = pendingRows;\n const flushedSize = pendingSize;\n\n let l = 0;\n for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n }\n // Insert the remaining rows individually.\n for (; pendingRows > 0; pendingRows--) {\n insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n }\n for (let i = 0; i < flushedRows; i++) {\n // Reuse the array and unreference the values to allow GC.\n // This is faster than allocating a new array every time.\n pendingValues[i] = undefined as unknown as LiteValueType;\n }\n pendingSize = 0;\n status.rows += flushedRows;\n\n const elapsed = performance.now() - start;\n flushTime += elapsed;\n lc.debug?.(\n `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n );\n }\n\n lc.info?.(`Starting copy stream of ${tableName}:`, select);\n const pgParsers = await getTypeParsers(dbClient, {returnJsonAsString: true});\n const parsers = columnSpecs.map(c => {\n const pgParse = pgParsers.getTypeParser(c.typeOID);\n return (val: string) =>\n liteValue(\n pgParse(val) as PostgresValueType,\n c.dataType,\n JSON_STRINGIFIED,\n );\n });\n\n const tsvParser = new TsvParser();\n let col = 0;\n\n await pipeline(\n await from.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n new Writable({\n highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n write(\n chunk: Buffer,\n _encoding: string,\n callback: (error?: Error) => void,\n ) {\n try {\n for (const text of tsvParser.parse(chunk)) {\n pendingSize += text === null ? 4 : text.length;\n pendingValues[pendingRows * valuesPerRow + col] =\n text === null ? null : parsers[col](text);\n\n if (++col === parsers.length) {\n col = 0;\n if (\n ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow \|\|\n pendingSize >= BUFFERED_SIZE_THRESHOLD\n ) {\n flush();\n }\n }\n }\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n\n final: (callback: (error?: Error) => void) => {\n try {\n flush();\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n }),\n );\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Finished copying ${status.rows} rows into ${tableName} ` +\n `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n );\n return {rows: status.rows, flushTime};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmEA,eAAsB,YACpB,IACA,OACA,IACA,aACA,aACA,SACA;AACA,KAAI,CAAC,0BAA0B,KAAK,MAAM,MAAM,CAC9C,OAAM,IAAI,MACR,2FACD;CAEH,MAAM,EAAC,kBAAkB,gBAAe;CACxC,MAAM,eAAe,cAAc,MAAM,YAAY,SAAS,GAAG;CACjE,MAAM,MAAM,SAAS,IAAI,YAAY;CACrC,MAAM,qBAAqB,SAAS,IAAI,aAAa;GAClD,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CAAC;CACF,MAAM,WAAW,mBAAmB,MAAM;CAC1C,MAAM,kBAAkB,2BAA2B,sBACjD,GACD,CAAC,QAAQ,IAAI,eAAe;AAC7B,KAAI;AACF,QAAM,oBAAoB,IAAI;EAE9B,MAAM,EAAC,iBAAgB,MAAM,sBAAsB,IAAI,KAAK,MAAM;AAClE,KAAG,OAAO,wCAAwC,aAAa,GAAG;EAElE,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,KAAG,OAAO,kCAAkC,SAAS,GAAG,OAAO;EAE/D,IAAI;AACJ,OAAK,IAAI,QAAQ,OAAQ,QAAQ,MAC/B,KAAI;AACF,UAAO,MAAM,sBAAsB,IAAI,oBAAoB,SAAS;AACpE;WACO,GAAG;AACV,OAAI,SAAS,aAAa,SAAS,eAAe;AAChD,QAAI,EAAE,SAAS,2BAA2B;AAKxC,WAAM,GAAG;AACT,QAAG,OAAO,8CAA8C;AACxD;;AAEF,QAAI,EAAE,SAAS,iCAAiC;KAG9C,MAAM,UAAU,MAAM,GAAqB;;;uCAFpB,0BAA0B,MAAM,CAKb;AAC1C,SAAI,QAAQ,QAAQ;AAClB,SAAG,OACD,uCAAuC,QAAQ,KAAK,EAAC,WAAU,KAAK,IACpE,EACD;AACD;;AAEF,QAAG,QAAQ,oCAAoC,EAAE;;;AAGrD,SAAM;;EAGV,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAAO;EACzD,MAAM,iBAAiB,qBAAqB,IAAI;AAEhD,uBAAqB,IAAI,cAAc,gBAAgB,QAAQ;EAG/D,MAAM,QAAQ,YAAY,KAAK;EAE/B,MAAM,YAAY,MAAM,MACtB,KACA,OAAM,OAAM;AACV,SAAM,GAAG,OAAgB,6BAA6B,SAAS,GAAG;AAClE,UAAO,mBAAmB,IAAI,aAAa;KAE7C,EAAC,MAAM,UAAc,CACtB;AAED,uBAAqB,IAAI,UAAU;EAGnC,MAAM,EAAC,QAAQ,YAAW;EAC1B,MAAM,YAAY,OAAO;AACzB,MAAI,UAAU,KAAK,WAAW,mBAAmB,UAC/C,IAAG,OACD,8CAA8C,iBAAiB,MAC1D,UAAU,gDAChB;EAEH,MAAM,aACJ,UAAU,KAAK,UACX,YACA,KAAK,IAAI,kBAAkB,UAAU;EAE3C,MAAM,WAAW,SAAS,IAAI,aAAa;GACzC,KAAK;GACL,YAAY,GAAE,qBAAqB,4BAA2B;IAC7D,iBAAiB;GACnB,CAAC;EACF,MAAM,UAAU,sBACd,IACA,UACA,UACA,YACA,UACD;AACD,MAAI;AACF,oBAAiB,IAAI,QAAQ,eAAe;GAC5C,MAAM,YAAY,MAAM,QAAQ,IAC9B,OAAO,KAAI,SACT,QAAQ,iBAAiB,IAAI,OAC3B,wBAAwB,IAAI,IAAI,KAAK,CACtC,CACF,CACF;AACD,mBAAgB,QACd,IACA,gBACA,WAAW,UAAU,8BAA8B,kBACnD,YACO,EAAC,gBAAgB,UAAU,KAAK,EAAC,aAAY,OAAO,EAAC,EAC7D;AAEI,iBAAc,OAAO;GAC1B,MAAM,YAAY,MAAM,QAAQ,IAC9B,UAAU,KAAI,UACZ,QAAQ,iBAAiB,IAAI,OAC3B,KAAK,IAAI,OAAO,UAAU,IAAI,GAAG,CAClC,CACF,CACF;AACI,iBAAc,eAAe,IAAI,eAAe;AACrD,WAAQ,SAAS;GAEjB,MAAM,QAAQ,UAAU,QACrB,KAAK,UAAU;IACd,MAAM,IAAI,OAAO,KAAK;IACtB,WAAW,IAAI,YAAY,KAAK;IACjC,GACD;IAAC,MAAM;IAAG,WAAW;IAAE,CACxB;AAED,mBAAgB,QACd,IACA,YACA,YAAY,QAAQ,OAAO,WAC3B,IACD;GACD,MAAM,aAAa,YAAY,KAAK;AACpC,qBAAkB,IAAI,QAAQ;GAC9B,MAAM,QAAQ,YAAY,KAAK,GAAG;AAClC,MAAG,OAAO,oBAAoB,MAAM,QAAQ,EAAE,CAAC,MAAM;AAErD,SAAM,WACJ,KACA,OACA,UACA,gBACA,WACA,QACD;GAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,MAAG,OACD,UAAU,MAAM,KAAK,gBAAgB,CAAC,WAAW,UAAU,aAAa,aAAa,SAAS,IAAI,WACrF,MAAM,UAAU,QAAQ,EAAE,CAAC,WAAW,MAAM,QAAQ,EAAE,CAAC,WAAW,QAAQ,QAAQ,EAAE,CAAC,MACnG;YACO;AAEH,YAAS,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,0BAA0B,EAAE,CAAC;;UAEjE,GAAG;AAIV,KAAG,OAAO,6BAA6B,YAAY,EAAE;AACrD,QAAM,GAAG;;4BAEe,SAAS;MAC/B,OAAM,MAAK,GAAG,OAAO,mCAAmC,YAAY,EAAE,CAAC;AACzE,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;AACtB,QAAM,mBAAmB,KAAK;AAC9B,QAAM,IAAI,KAAK;;;AAInB,eAAe,oBAAoB,KAAiB;CAClD,MAAM,EAAC,UAAU,aACf,MAAM,GAA0C;;;KAIhD;AAEF,KAAI,aAAa,UACf,OAAM,IAAI,MACR,uEAAuE,SAAS,GACjF;AAEH,KAAI,UAAU,KACZ,OAAM,IAAI,MACR,sDAAsD,QAAQ,IAC/D;;AAIL,eAAe,sBACb,IACA,KACA,OACA,WAAW,MACwB;CACnC,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,IAAG,OAAO,oCAAoC,SAAS,GAAG,OAAO;AAEjE,OAAM,kBAAkB,IAAI,KAAK,MAAM;CACvC,MAAM,EAAC,iBAAgB,MAAM,uBAAuB,KAAK,MAAM;AAE/D,KAAI,UAAU;EACZ,IAAI,QAAQ;EACZ,MAAM,0BAA0B,aAAa,QAC3C,MAAK,CAAC,EAAE,WAAW,IAAI,CACxB;EACD,MAAM,SAAS,MAAM,GAAG;4DACgC,IAAI,aAAa,CAAC;QACtE,QAAQ;AACZ,MAAI,OAAO,WAAW,aAAa,OACjC,IAAG,OACD,iCAAiC,aAAa,kBACxC,OAAO,MAAM,CAAC,cACrB;WAED,CAAC,OAAO,IAAI,IAAI,MAAM,aAAa,EAAE,IAAI,IAAI,wBAAwB,CAAC,CAEtE,IAAG,OACD,2BAA2B,MAAM,aAAa,sCAC3B,wBAAwB,cAC5C;MAED,SAAQ;AAEV,MAAI,CAAC,OAAO;AACV,SAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,SAAS,CAAC;AACxD,UAAO,sBAAsB,IAAI,KAAK,OAAO,MAAM;;;AAGvD,QAAO,EAAC,cAAa;;AAGvB,SAAS,sBACP,IACA,IACA,UACA,YACA,WACiB;CACjB,MAAM,EAAC,SAAQ,eAAe,SAAS;CACvC,MAAM,eAAe,IAAI,gBACvB,IACA,UACA,MACA,KAAA,GACA,WACD;AACD,cAAa,IAAI,GAAG;AAEpB,IAAG,OAAO,WAAW,WAAW,mBAAmB,UAAU,SAAS;AAEtE,KAAI,SAAS,QAAQ,SAAS,KAAK,GAAG,GACpC,IAAG,OACD,mUAMD;AAEH,QAAO;;AAcT,eAAsB,sBACpB,IACA,SACA,UAC0B;CAC1B,MAAM,QACJ,MAAM,QAAQ,OACJ,4BAA4B,SAAS,oBAC9C,EACD;AACF,IAAG,OAAO,4BAA4B,YAAY,KAAK;AACvD,QAAO;;AAGT,SAAS,iBACP,IACA,QACA,gBACA;CAGA,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,GAAG,CAAC;AAChE,MAAK,MAAM,KAAK,QAAQ;AACtB,KAAG,KAAK,yBAAyB,kBAAkB,GAAG,eAAe,CAAC,CAAC;EACvE,MAAM,YAAY,cAAc,EAAE;AAClC,OAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,EAAE,QAAQ,CACxD,gBAAe,OAAO,WAAW,SAAS,QAAQ;;;AAKxD,SAAS,kBAAkB,IAAc,SAAsB;AAC7D,MAAK,MAAM,SAAS,QAClB,IAAG,KAAK,yBAAyB,uBAAuB,MAAM,CAAC,CAAC;;AAUpE,IAAM,KAAK,OAAO;AAClB,IAAM,oBAAoB;AAC1B,IAAM,0BAA0B,IAAI;AAQpC,SAAgB,uBACd,OACA,MACoB;CACpB,MAAM,mBAAmB,OAAO,OAAO,MAAM,aAAa,CACvD,KAAK,EAAC,gBAAe,UAAU,CAC/B,QAAO,MAAK,CAAC,CAAC,EAAE;CACnB,MAAM,QACJ,iBAAiB,WAAW,IACxB,KACQ,SAAS,iBAAiB,KAAK,OAAO;CACpD,MAAM,YAAoB,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG;CACxE,MAAM,aAAa,IAAI,KAAK,KAAI,QAAO,+BAA+B,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;AAMnG,QALc;EACZ,QAAgB,UAAU,KAAK,IAAI,GAAG,CAAC,KAAK,IAAI,CAAC,GAAG;EACpD,cAAsB,kCAAkC;EACxD,eAAuB,UAAU,WAAW,mBAAmB;EAChE;;AASH,eAAe,wBACb,IACA,KACA,MACwB;CACxB,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,QAAQ,cAAc,KAAK;CACjC,MAAM,UAAU,OAAO,KAAK,KAAK,QAAQ;CACzC,MAAM,QAAQ,uBAAuB,MAAM,QAAQ;CACnD,MAAM,aAAa,IAChB,OAA8B,MAAM,aAAa,CACjD,SAAS;CACZ,MAAM,cAAc,IACjB,OAA+B,MAAM,cAAc,CACnD,SAAS;CAEZ,MAAM,QAAuB;EAC3B;EACA,QAAQ;GACN;GACA;GACA,MAAM;GACN,WAAW,QAAQ,MAAM,YAAY,GAAG,UAAU;GAClD,YAAY,QAAQ,MAAM,aAAa,GAAG,WAAW;GACtD;EACF;CACD,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,IAAG,OAAO,uCAAuC,MAAM,IAAI,QAAQ,OAAO,EACxE,OAAO,MAAM,QACd,CAAC;AACF,QAAO;;AAGT,eAAe,KACb,IACA,EAAC,MAAM,OAAO,UACd,UACA,MACA,IACA;CACA,MAAM,QAAQ,YAAY,KAAK;CAC/B,IAAI,YAAY;CAEhB,MAAM,YAAY,cAAc,MAAM;CACtC,MAAM,iBAAiB,OAAO,QAAQ,MAAM,QAAQ;CAEpD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,EAAE;CAClD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,UAAU,KAAK;CAC/D,MAAM,mBAAmB,YAAY,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI;CAG9D,MAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,EAAE,CAAC,MAAM;CACzE,MAAM,YAAoB;mBACT,UAAU,KAAK,iBAAiB,WAAW;CAC5D,MAAM,aAAa,GAAG,QAAQ,UAAU;CAExC,MAAM,kBAAkB,GAAG,QACzB,YAAY,IAAI,YAAY,OAAA,GAA6B,CAC1D;CAED,MAAM,EAAC,WAAU,uBAAuB,OAAO,YAAY;CAC3D,MAAM,eAAe,YAAY;CACjC,MAAM,iBAAiB,eAAA;CAGvB,MAAM,gBAAiC,MAAM,KAAK,EAChD,QAAQ,oBAAoB,cAC7B,CAAC;CACF,IAAI,cAAc;CAClB,IAAI,cAAc;CAElB,SAAS,QAAQ;EACf,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,cAAc;EACpB,MAAM,cAAc;EAEpB,IAAI,IAAI;AACR,SAAO,cAAA,IAAiC,eAAA,GACtC,iBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,eAAgB,CAAC;AAGpE,SAAO,cAAc,GAAG,cACtB,YAAW,IAAI,cAAc,MAAM,GAAI,KAAK,aAAc,CAAC;AAE7D,OAAK,IAAI,IAAI,GAAG,IAAI,aAAa,IAG/B,eAAc,KAAK,KAAA;AAErB,gBAAc;AACd,SAAO,QAAQ;EAEf,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,eAAa;AACb,KAAG,QACD,WAAW,YAAY,GAAG,UAAU,SAAS,YAAY,aAAa,QAAQ,QAAQ,EAAE,CAAC,KAC1F;;AAGH,IAAG,OAAO,2BAA2B,UAAU,IAAI,OAAO;CAC1D,MAAM,YAAY,MAAM,eAAe,UAAU,EAAC,oBAAoB,MAAK,CAAC;CAC5E,MAAM,UAAU,YAAY,KAAI,MAAK;EACnC,MAAM,UAAU,UAAU,cAAc,EAAE,QAAQ;AAClD,UAAQ,QACN,UACE,QAAQ,IAAI,EACZ,EAAE,UAAA,IAEH;GACH;CAEF,MAAM,YAAY,IAAI,WAAW;CACjC,IAAI,MAAM;AAEV,OAAM,WACJ,MAAM,KAAK,OAAO,SAAS,OAAO,aAAa,CAAC,UAAU,EAC1D,IAAI,SAAS;EACX,eAAe;EAEf,MACE,OACA,WACA,UACA;AACA,OAAI;AACF,SAAK,MAAM,QAAQ,UAAU,MAAM,MAAM,EAAE;AACzC,oBAAe,SAAS,OAAO,IAAI,KAAK;AACxC,mBAAc,cAAc,eAAe,OACzC,SAAS,OAAO,OAAO,QAAQ,KAAK,KAAK;AAE3C,SAAI,EAAE,QAAQ,QAAQ,QAAQ;AAC5B,YAAM;AACN,UACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,wBAEf,QAAO;;;AAIb,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAI3D,QAAQ,aAAsC;AAC5C,OAAI;AACF,WAAO;AACP,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAG5D,CAAC,CACH;CAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,IAAG,OACD,oBAAoB,OAAO,KAAK,aAAa,UAAU,WAC1C,UAAU,QAAQ,EAAE,CAAC,eAAe,QAAQ,QAAQ,EAAE,CAAC,OACrE;AACD,QAAO;EAAC,MAAM,OAAO;EAAM;EAAU"}
1	+ {"version":3,"file":"initial-sync.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"sourcesContent":["import {\n PG_CONFIGURATION_LIMIT_EXCEEDED,\n PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {platform} from 'node:os';\nimport {Writable} from 'node:stream';\nimport {pipeline} from 'node:stream/promises';\nimport postgres from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport type {DownloadStatus} from '../../../../../zero-events/src/status.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {\n createLiteIndexStatement,\n createLiteTableStatement,\n} from '../../../db/create.ts';\nimport * as Mode from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {\n mapPostgresToLite,\n mapPostgresToLiteIndex,\n} from '../../../db/pg-to-lite.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport type {IndexSpec, PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {\n JSON_STRINGIFIED,\n liteValue,\n type LiteValueType,\n} from '../../../types/lite.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport {PG_15, PG_17} from '../../../types/pg-versions.ts';\nimport {\n pgClient,\n type PostgresDB,\n type PostgresTransaction,\n type PostgresValueType,\n} from '../../../types/pg.ts';\nimport {CpuProfiler} from '../../../types/profiler.ts';\nimport type {ShardConfig} from '../../../types/shards.ts';\nimport {ALLOWED_APP_ID_CHARACTERS} from '../../../types/shards.ts';\nimport {id} from '../../../types/sql.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {ColumnMetadataStore} from '../../replicator/schema/column-metadata.ts';\nimport {initReplicationState} from '../../replicator/schema/replication-state.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport {\n addReplica,\n dropShard,\n getInternalShardConfig,\n newReplicationSlot,\n replicationSlotExpression,\n validatePublications,\n} from './schema/shard.ts';\n\nexport type InitialSyncOptions = {\n tableCopyWorkers: number;\n profileCopy?: boolean \| undefined;\n textCopy?: boolean \| undefined;\n replicationSlotFailover?: boolean \| undefined;\n};\n\n/** Server context to store with the initial sync metadata for debugging. /\nexport type ServerContext = JSONObject;\n\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n syncOptions: InitialSyncOptions,\n context: ServerContext,\n) {\n if (!ALLOWED_APP_ID_CHARACTERS.test(shard.appID)) {\n throw new Error(\n 'The App ID may only consist of lower-case letters, numbers, and the underscore character',\n );\n }\n const {\n tableCopyWorkers,\n profileCopy,\n textCopy = false,\n replicationSlotFailover = false,\n } = syncOptions;\n const copyProfiler = profileCopy ? await CpuProfiler.connect() : null;\n const sql = pgClient(lc, upstreamURI);\n const replicationSession = pgClient(lc, upstreamURI, {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n });\n const slotName = newReplicationSlot(shard);\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(\n tx,\n ).publish(lc, 'Initializing');\n try {\n const pgVersion = await checkUpstreamConfig(sql);\n\n const {publications} = await ensurePublishedTables(lc, sql, shard);\n lc.info?.(`Upstream is setup with publications [${publications}]`);\n\n const {database, host} = sql.options;\n lc.info?.(`opening replication session to ${database}@${host}`);\n\n let slot: ReplicationSlot;\n for (let first = true; ; first = false) {\n try {\n slot = await createReplicationSlot(\n lc,\n replicationSession,\n slotName,\n replicationSlotFailover && pgVersion >= PG_17,\n );\n break;\n } catch (e) {\n if (first && e instanceof postgres.PostgresError) {\n if (e.code === PG_INSUFFICIENT_PRIVILEGE) {\n // Some Postgres variants (e.g. Google Cloud SQL) require that\n // the user have the REPLICATION role in order to create a slot.\n // Note that this must be done by the upstreamDB connection, and\n // does not work in the replicationSession itself.\n await sql`ALTER ROLE current_user WITH REPLICATION`;\n lc.info?.(`Added the REPLICATION role to database user`);\n continue;\n }\n if (e.code === PG_CONFIGURATION_LIMIT_EXCEEDED) {\n const slotExpression = replicationSlotExpression(shard);\n\n const dropped = await sql<{slot: string}[]>`\n SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n FROM pg_replication_slots\n WHERE slot_name LIKE ${slotExpression} AND NOT active`;\n if (dropped.length) {\n lc.warn?.(\n `Dropped inactive replication slots: ${dropped.map(({slot}) => slot)}`,\n e,\n );\n continue;\n }\n lc.error?.(`Unable to drop replication slots`, e);\n }\n }\n throw e;\n }\n }\n const {snapshot_name: snapshot, consistent_point: lsn} = slot;\n const initialVersion = toStateVersionString(lsn);\n\n initReplicationState(tx, publications, initialVersion, context);\n\n // Run up to MAX_WORKERS to copy of tables at the replication slot's snapshot.\n const start = performance.now();\n // Retrieve the published schema at the consistent_point.\n const published = await runTx(\n sql,\n async tx => {\n await tx.unsafe(/ sql/ `SET TRANSACTION SNAPSHOT '${snapshot}'`);\n return getPublicationInfo(tx, publications);\n },\n {mode: Mode.READONLY},\n );\n // Note: If this throws, initial-sync is aborted.\n validatePublications(lc, published);\n\n // Now that tables have been validated, kick off the copiers.\n const {tables, indexes} = published;\n const numTables = tables.length;\n if (platform() === 'win32' && tableCopyWorkers < numTables) {\n lc.warn?.(\n `Increasing the number of copy workers from ${tableCopyWorkers} to ` +\n `${numTables} to work around a Node/Postgres connection bug`,\n );\n }\n const numWorkers =\n platform() === 'win32'\n ? numTables\n : Math.min(tableCopyWorkers, numTables);\n\n const copyPool = pgClient(lc, upstreamURI, {\n max: numWorkers,\n connection: {['application_name']: 'initial-sync-copy-worker'},\n ['max_lifetime']: 120 60, // set a long (2h) limit for COPY streaming\n });\n const copiers = startTableCopyWorkers(\n lc,\n copyPool,\n snapshot,\n numWorkers,\n numTables,\n );\n try {\n createLiteTables(tx, tables, initialVersion);\n const downloads = await Promise.all(\n tables.map(spec =>\n copiers.processReadTask((db, lc) =>\n getInitialDownloadState(lc, db, spec),\n ),\n ),\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying ${numTables} upstream tables at version ${initialVersion}`,\n 5000,\n () => ({downloadStatus: downloads.map(({status}) => status)}),\n );\n\n void copyProfiler?.start();\n const rowCounts = await Promise.all(\n downloads.map(table =>\n copiers.processReadTask((db, lc) =>\n copy(lc, table, copyPool, db, tx, textCopy),\n ),\n ),\n );\n void copyProfiler?.stopAndDispose(lc, 'initial-copy');\n copiers.setDone();\n\n const total = rowCounts.reduce(\n (acc, curr) => ({\n rows: acc.rows + curr.rows,\n flushTime: acc.flushTime + curr.flushTime,\n }),\n {rows: 0, flushTime: 0},\n );\n\n statusPublisher.publish(\n lc,\n 'Indexing',\n `Creating ${indexes.length} indexes`,\n 5000,\n );\n const indexStart = performance.now();\n createLiteIndices(tx, indexes);\n const index = performance.now() - indexStart;\n lc.info?.(`Created indexes (${index.toFixed(3)} ms)`);\n\n await addReplica(\n sql,\n shard,\n slotName,\n initialVersion,\n published,\n context,\n );\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Synced ${total.rows.toLocaleString()} rows of ${numTables} tables in ${publications} up to ${lsn} ` +\n `(flush: ${total.flushTime.toFixed(3)}, index: ${index.toFixed(3)}, total: ${elapsed.toFixed(3)} ms)`,\n );\n } finally {\n // All meaningful errors are handled at the processReadTask() call site.\n void copyPool.end().catch(e => lc.warn?.(`Error closing copyPool`, e));\n }\n } catch (e) {\n // If initial-sync did not succeed, make a best effort to drop the\n // orphaned replication slot to avoid running out of slots in\n // pathological cases that result in repeated failures.\n lc.warn?.(`dropping replication slot ${slotName}`, e);\n await sql`\n SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n WHERE slot_name = ${slotName};\n `.catch(e => lc.warn?.(`Unable to drop replication slot ${slotName}`, e));\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n await replicationSession.end();\n await sql.end();\n }\n}\n\nasync function checkUpstreamConfig(sql: PostgresDB) {\n const {walLevel, version} = (\n await sql<{walLevel: string; version: number}[]>`\n SELECT current_setting('wal_level') as \"walLevel\", \n current_setting('server_version_num') as \"version\";\n `\n )[0];\n\n if (walLevel !== 'logical') {\n throw new Error(\n `Postgres must be configured with \"wal_level = logical\" (currently: \"${walLevel})`,\n );\n }\n if (version < PG_15) {\n throw new Error(\n `Must be running Postgres 15 or higher (currently: \"${version}\")`,\n );\n }\n return version;\n}\n\nasync function ensurePublishedTables(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardConfig,\n validate = true,\n): Promise<{publications: string[]}> {\n const {database, host} = sql.options;\n lc.info?.(`Ensuring upstream PUBLICATION on ${database}@${host}`);\n\n await ensureShardSchema(lc, sql, shard);\n const {publications} = await getInternalShardConfig(sql, shard);\n\n if (validate) {\n let valid = false;\n const nonInternalPublications = publications.filter(\n p => !p.startsWith('_'),\n );\n const exists = await sql`\n SELECT pubname FROM pg_publication WHERE pubname IN ${sql(publications)}\n `.values();\n if (exists.length !== publications.length) {\n lc.warn?.(\n `some configured publications [${publications}] are missing: ` +\n `[${exists.flat()}]. resyncing`,\n );\n } else if (\n !equals(new Set(shard.publications), new Set(nonInternalPublications))\n ) {\n lc.warn?.(\n `requested publications [${shard.publications}] differ from previous` +\n `publications [${nonInternalPublications}]. resyncing`,\n );\n } else {\n valid = true;\n }\n if (!valid) {\n await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n return ensurePublishedTables(lc, sql, shard, false);\n }\n }\n return {publications};\n}\n\nfunction startTableCopyWorkers(\n lc: LogContext,\n db: PostgresDB,\n snapshot: string,\n numWorkers: number,\n numTables: number,\n): TransactionPool {\n const {init} = importSnapshot(snapshot);\n const tableCopiers = new TransactionPool(\n lc,\n Mode.READONLY,\n init,\n undefined,\n numWorkers,\n );\n tableCopiers.run(db);\n\n lc.info?.(`Started ${numWorkers} workers to copy ${numTables} tables`);\n\n if (parseInt(process.versions.node) < 22) {\n lc.warn?.(\n `\\n\\n\\n` +\n `Older versions of Node have a bug that results in an unresponsive\\n` +\n `Postgres connection after running certain combinations of COPY commands.\\n` +\n `If initial sync hangs, run zero-cache with Node v22+. This has the additional\\n` +\n `benefit of being consistent with the Node version run in the production container image.` +\n `\\n\\n\\n`,\n );\n }\n return tableCopiers;\n}\n\n// Row returned by `CREATE_REPLICATION_SLOT`\ntype ReplicationSlot = {\n slot_name: string;\n consistent_point: string;\n snapshot_name: string;\n output_plugin: string;\n};\n\n// Note: The replication connection does not support the extended query protocol,\n// so all commands must be sent using sql.unsafe(). This is technically safe\n// because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n lc: LogContext,\n session: postgres.Sql,\n slotName: string,\n // Note: must be false if pgVersion < PG_17. Caller must verify.\n failover = false,\n): Promise<ReplicationSlot> {\n const [slot] = failover\n ? await session.unsafe<ReplicationSlot[]>(\n /sql/ `CREATE_REPLICATION_SLOT \"${slotName}\" LOGICAL pgoutput (FAILOVER)`,\n )\n : await session.unsafe<ReplicationSlot[]>(\n /sql/ `CREATE_REPLICATION_SLOT \"${slotName}\" LOGICAL pgoutput`,\n );\n lc.info?.(`Created replication slot ${slotName}`, slot);\n return slot;\n}\n\nfunction createLiteTables(\n tx: Database,\n tables: PublishedTableSpec[],\n initialVersion: string,\n) {\n // TODO: Figure out how to reuse the ChangeProcessor here to avoid\n // duplicating the ColumnMetadata logic.\n const columnMetadata = must(ColumnMetadataStore.getInstance(tx));\n for (const t of tables) {\n tx.exec(createLiteTableStatement(mapPostgresToLite(t, initialVersion)));\n const tableName = liteTableName(t);\n for (const [colName, colSpec] of Object.entries(t.columns)) {\n columnMetadata.insert(tableName, colName, colSpec);\n }\n }\n}\n\nfunction createLiteIndices(tx: Database, indices: IndexSpec[]) {\n for (const index of indices) {\n tx.exec(createLiteIndexStatement(mapPostgresToLiteIndex(index)));\n }\n}\n\n// Verified empirically that batches of 50 seem to be the sweet spot,\n// similar to the report in https://sqlite.org/forum/forumpost/8878a512d3652655\n//\n// Exported for testing.\nexport const INSERT_BATCH_SIZE = 50;\n\nconst MB = 1024 * 1024;\nconst MAX_BUFFERED_ROWS = 10_000;\nconst BUFFERED_SIZE_THRESHOLD = 8 * MB;\n\nexport type DownloadStatements = {\n select: string;\n getTotalRows: string;\n getTotalBytes: string;\n};\n\nexport function makeDownloadStatements(\n table: PublishedTableSpec,\n cols: string[],\n): DownloadStatements {\n const filterConditions = Object.values(table.publications)\n .map(({rowFilter}) => rowFilter)\n .filter(f => !!f); // remove nulls\n const where =\n filterConditions.length === 0\n ? ''\n : /sql/ `WHERE ${filterConditions.join(' OR ')}`;\n const fromTable = /sql/ `FROM ${id(table.schema)}.${id(table.name)} ${where}`;\n const totalBytes = `(${cols.map(col => `SUM(COALESCE(pg_column_size(${id(col)}), 0))`).join(' + ')})`;\n const stmts = {\n select: /sql/ `SELECT ${cols.map(id).join(',')} ${fromTable}`,\n getTotalRows: /sql/ `SELECT COUNT() AS \"totalRows\" ${fromTable}`,\n getTotalBytes: /sql/ `SELECT ${totalBytes} AS \"totalBytes\" ${fromTable}`,\n };\n return stmts;\n}\n\ntype DownloadState = {\n spec: PublishedTableSpec;\n status: DownloadStatus;\n};\n\nasync function getInitialDownloadState(\n lc: LogContext,\n sql: PostgresDB,\n spec: PublishedTableSpec,\n): Promise<DownloadState> {\n const start = performance.now();\n const table = liteTableName(spec);\n const columns = Object.keys(spec.columns);\n const stmts = makeDownloadStatements(spec, columns);\n const rowsResult = sql\n .unsafe<{totalRows: bigint}[]>(stmts.getTotalRows)\n .execute();\n const bytesResult = sql\n .unsafe<{totalBytes: bigint}[]>(stmts.getTotalBytes)\n .execute();\n\n const state: DownloadState = {\n spec,\n status: {\n table,\n columns,\n rows: 0,\n totalRows: Number((await rowsResult)[0].totalRows),\n totalBytes: Number((await bytesResult)[0].totalBytes),\n },\n };\n const elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(`Computed initial download state for ${table} (${elapsed} ms)`, {\n state: state.status,\n });\n return state;\n}\n\nfunction copy(\n lc: LogContext,\n {spec: table, status}: DownloadState,\n dbClient: PostgresDB,\n from: PostgresTransaction,\n to: Database,\n textCopy: boolean,\n) {\n if (textCopy) {\n return copyText(lc, table, status, dbClient, from, to);\n }\n return copyBinary(lc, table, status, from, to);\n}\n\nasync function copyBinary(\n lc: LogContext,\n table: PublishedTableSpec,\n status: DownloadStatus,\n from: PostgresTransaction,\n to: Database,\n) {\n const start = performance.now();\n let flushTime = 0;\n\n const tableName = liteTableName(table);\n const orderedColumns = Object.entries(table.columns);\n\n const columnNames = orderedColumns.map(([c]) => c);\n const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n const valuesSql =\n columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n const insertSql = /sql/ `\n INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n const insertStmt = to.prepare(insertSql);\n const insertBatchStmt = to.prepare(\n insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n );\n\n // Build SELECT with ::text casts for columns without a known binary decoder.\n const filterConditions = Object.values(table.publications)\n .map(({rowFilter}) => rowFilter)\n .filter(f => !!f);\n const where =\n filterConditions.length === 0\n ? ''\n : /sql/ `WHERE ${filterConditions.join(' OR ')}`;\n const fromTable = /sql/ `FROM ${id(table.schema)}.${id(table.name)} ${where}`;\n const selectColumns = orderedColumns.map(([name, spec]) =>\n hasBinaryDecoder(spec) ? id(name) : `${id(name)}::text`,\n );\n const select = /sql/ `SELECT ${selectColumns.join(',')} ${fromTable}`;\n\n const decoders = orderedColumns.map(([, spec]) =>\n hasBinaryDecoder(spec) ? makeBinaryDecoder(spec) : textCastDecoder,\n );\n\n const valuesPerRow = columnSpecs.length;\n const valuesPerBatch = valuesPerRow INSERT_BATCH_SIZE;\n\n const pendingValues: LiteValueType[] = Array.from({\n length: MAX_BUFFERED_ROWS * valuesPerRow,\n });\n let pendingRows = 0;\n let pendingSize = 0;\n\n function flush() {\n const start = performance.now();\n const flushedRows = pendingRows;\n const flushedSize = pendingSize;\n\n let l = 0;\n for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n }\n for (; pendingRows > 0; pendingRows--) {\n insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n }\n const flushedValues = flushedRows * valuesPerRow;\n for (let i = 0; i < flushedValues; i++) {\n pendingValues[i] = undefined as unknown as LiteValueType;\n }\n pendingSize = 0;\n status.rows += flushedRows;\n\n const elapsed = performance.now() - start;\n flushTime += elapsed;\n lc.debug?.(\n `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n );\n }\n\n const binaryParser = new BinaryCopyParser();\n let col = 0;\n\n lc.info?.(`Starting binary copy stream of ${tableName}:`, select);\n\n await pipeline(\n await from\n .unsafe(`COPY (${select}) TO STDOUT WITH (FORMAT binary)`)\n .readable(),\n new Writable({\n highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n write(\n chunk: Buffer,\n _encoding: string,\n callback: (error?: Error) => void,\n ) {\n try {\n for (const fieldBuf of binaryParser.parse(chunk)) {\n pendingSize += fieldBuf === null ? 4 : fieldBuf.length;\n pendingValues[pendingRows * valuesPerRow + col] =\n fieldBuf === null ? null : decoders[col](fieldBuf);\n\n if (++col === decoders.length) {\n col = 0;\n if (\n ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow \|\|\n pendingSize >= BUFFERED_SIZE_THRESHOLD\n ) {\n flush();\n }\n }\n }\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n\n final: (callback: (error?: Error) => void) => {\n try {\n flush();\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n }),\n );\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Finished copying ${status.rows} rows into ${tableName} ` +\n `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n );\n return {rows: status.rows, flushTime};\n}\n\nasync function copyText(\n lc: LogContext,\n table: PublishedTableSpec,\n status: DownloadStatus,\n dbClient: PostgresDB,\n from: PostgresTransaction,\n to: Database,\n) {\n const start = performance.now();\n let flushTime = 0;\n\n const tableName = liteTableName(table);\n const orderedColumns = Object.entries(table.columns);\n\n const columnNames = orderedColumns.map(([c]) => c);\n const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n const valuesSql =\n columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n const insertSql = /sql/ `\n INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n const insertStmt = to.prepare(insertSql);\n const insertBatchStmt = to.prepare(\n insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n );\n\n const {select} = makeDownloadStatements(table, columnNames);\n const valuesPerRow = columnSpecs.length;\n const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;\n\n const pendingValues: LiteValueType[] = Array.from({\n length: MAX_BUFFERED_ROWS * valuesPerRow,\n });\n let pendingRows = 0;\n let pendingSize = 0;\n\n function flush() {\n const start = performance.now();\n const flushedRows = pendingRows;\n const flushedSize = pendingSize;\n\n let l = 0;\n for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n }\n for (; pendingRows > 0; pendingRows--) {\n insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n }\n const flushedValues = flushedRows * valuesPerRow;\n for (let i = 0; i < flushedValues; i++) {\n pendingValues[i] = undefined as unknown as LiteValueType;\n }\n pendingSize = 0;\n status.rows += flushedRows;\n\n const elapsed = performance.now() - start;\n flushTime += elapsed;\n lc.debug?.(\n `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n );\n }\n\n lc.info?.(`Starting text copy stream of ${tableName}:`, select);\n const pgParsers = await getTypeParsers(dbClient, {returnJsonAsString: true});\n const parsers = columnSpecs.map(c => {\n const pgParse = pgParsers.getTypeParser(c.typeOID);\n return (val: string) =>\n liteValue(\n pgParse(val) as PostgresValueType,\n c.dataType,\n JSON_STRINGIFIED,\n );\n });\n\n const tsvParser = new TsvParser();\n let col = 0;\n\n await pipeline(\n await from.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n new Writable({\n highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n write(\n chunk: Buffer,\n _encoding: string,\n callback: (error?: Error) => void,\n ) {\n try {\n for (const text of tsvParser.parse(chunk)) {\n pendingSize += text === null ? 4 : text.length;\n pendingValues[pendingRows * valuesPerRow + col] =\n text === null ? null : parsers[col](text);\n\n if (++col === parsers.length) {\n col = 0;\n if (\n ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow \|\|\n pendingSize >= BUFFERED_SIZE_THRESHOLD\n ) {\n flush();\n }\n }\n }\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n\n final: (callback: (error?: Error) => void) => {\n try {\n flush();\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n }),\n );\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Finished copying ${status.rows} rows into ${tableName} ` +\n `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n );\n return {rows: status.rows, flushTime};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4EA,eAAsB,YACpB,IACA,OACA,IACA,aACA,aACA,SACA;AACA,KAAI,CAAC,0BAA0B,KAAK,MAAM,MAAM,CAC9C,OAAM,IAAI,MACR,2FACD;CAEH,MAAM,EACJ,kBACA,aACA,WAAW,OACX,0BAA0B,UACxB;CACJ,MAAM,eAAe,cAAc,MAAM,YAAY,SAAS,GAAG;CACjE,MAAM,MAAM,SAAS,IAAI,YAAY;CACrC,MAAM,qBAAqB,SAAS,IAAI,aAAa;GAClD,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CAAC;CACF,MAAM,WAAW,mBAAmB,MAAM;CAC1C,MAAM,kBAAkB,2BAA2B,sBACjD,GACD,CAAC,QAAQ,IAAI,eAAe;AAC7B,KAAI;EACF,MAAM,YAAY,MAAM,oBAAoB,IAAI;EAEhD,MAAM,EAAC,iBAAgB,MAAM,sBAAsB,IAAI,KAAK,MAAM;AAClE,KAAG,OAAO,wCAAwC,aAAa,GAAG;EAElE,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,KAAG,OAAO,kCAAkC,SAAS,GAAG,OAAO;EAE/D,IAAI;AACJ,OAAK,IAAI,QAAQ,OAAQ,QAAQ,MAC/B,KAAI;AACF,UAAO,MAAM,sBACX,IACA,oBACA,UACA,2BAA2B,aAAA,KAC5B;AACD;WACO,GAAG;AACV,OAAI,SAAS,aAAa,SAAS,eAAe;AAChD,QAAI,EAAE,SAAS,2BAA2B;AAKxC,WAAM,GAAG;AACT,QAAG,OAAO,8CAA8C;AACxD;;AAEF,QAAI,EAAE,SAAS,iCAAiC;KAG9C,MAAM,UAAU,MAAM,GAAqB;;;uCAFpB,0BAA0B,MAAM,CAKb;AAC1C,SAAI,QAAQ,QAAQ;AAClB,SAAG,OACD,uCAAuC,QAAQ,KAAK,EAAC,WAAU,KAAK,IACpE,EACD;AACD;;AAEF,QAAG,QAAQ,oCAAoC,EAAE;;;AAGrD,SAAM;;EAGV,MAAM,EAAC,eAAe,UAAU,kBAAkB,QAAO;EACzD,MAAM,iBAAiB,qBAAqB,IAAI;AAEhD,uBAAqB,IAAI,cAAc,gBAAgB,QAAQ;EAG/D,MAAM,QAAQ,YAAY,KAAK;EAE/B,MAAM,YAAY,MAAM,MACtB,KACA,OAAM,OAAM;AACV,SAAM,GAAG,OAAgB,6BAA6B,SAAS,GAAG;AAClE,UAAO,mBAAmB,IAAI,aAAa;KAE7C,EAAC,MAAM,UAAc,CACtB;AAED,uBAAqB,IAAI,UAAU;EAGnC,MAAM,EAAC,QAAQ,YAAW;EAC1B,MAAM,YAAY,OAAO;AACzB,MAAI,UAAU,KAAK,WAAW,mBAAmB,UAC/C,IAAG,OACD,8CAA8C,iBAAiB,MAC1D,UAAU,gDAChB;EAEH,MAAM,aACJ,UAAU,KAAK,UACX,YACA,KAAK,IAAI,kBAAkB,UAAU;EAE3C,MAAM,WAAW,SAAS,IAAI,aAAa;GACzC,KAAK;GACL,YAAY,GAAE,qBAAqB,4BAA2B;IAC7D,iBAAiB;GACnB,CAAC;EACF,MAAM,UAAU,sBACd,IACA,UACA,UACA,YACA,UACD;AACD,MAAI;AACF,oBAAiB,IAAI,QAAQ,eAAe;GAC5C,MAAM,YAAY,MAAM,QAAQ,IAC9B,OAAO,KAAI,SACT,QAAQ,iBAAiB,IAAI,OAC3B,wBAAwB,IAAI,IAAI,KAAK,CACtC,CACF,CACF;AACD,mBAAgB,QACd,IACA,gBACA,WAAW,UAAU,8BAA8B,kBACnD,YACO,EAAC,gBAAgB,UAAU,KAAK,EAAC,aAAY,OAAO,EAAC,EAC7D;AAEI,iBAAc,OAAO;GAC1B,MAAM,YAAY,MAAM,QAAQ,IAC9B,UAAU,KAAI,UACZ,QAAQ,iBAAiB,IAAI,OAC3B,KAAK,IAAI,OAAO,UAAU,IAAI,IAAI,SAAS,CAC5C,CACF,CACF;AACI,iBAAc,eAAe,IAAI,eAAe;AACrD,WAAQ,SAAS;GAEjB,MAAM,QAAQ,UAAU,QACrB,KAAK,UAAU;IACd,MAAM,IAAI,OAAO,KAAK;IACtB,WAAW,IAAI,YAAY,KAAK;IACjC,GACD;IAAC,MAAM;IAAG,WAAW;IAAE,CACxB;AAED,mBAAgB,QACd,IACA,YACA,YAAY,QAAQ,OAAO,WAC3B,IACD;GACD,MAAM,aAAa,YAAY,KAAK;AACpC,qBAAkB,IAAI,QAAQ;GAC9B,MAAM,QAAQ,YAAY,KAAK,GAAG;AAClC,MAAG,OAAO,oBAAoB,MAAM,QAAQ,EAAE,CAAC,MAAM;AAErD,SAAM,WACJ,KACA,OACA,UACA,gBACA,WACA,QACD;GAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,MAAG,OACD,UAAU,MAAM,KAAK,gBAAgB,CAAC,WAAW,UAAU,aAAa,aAAa,SAAS,IAAI,WACrF,MAAM,UAAU,QAAQ,EAAE,CAAC,WAAW,MAAM,QAAQ,EAAE,CAAC,WAAW,QAAQ,QAAQ,EAAE,CAAC,MACnG;YACO;AAEH,YAAS,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,0BAA0B,EAAE,CAAC;;UAEjE,GAAG;AAIV,KAAG,OAAO,6BAA6B,YAAY,EAAE;AACrD,QAAM,GAAG;;4BAEe,SAAS;MAC/B,OAAM,MAAK,GAAG,OAAO,mCAAmC,YAAY,EAAE,CAAC;AACzE,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;AACtB,QAAM,mBAAmB,KAAK;AAC9B,QAAM,IAAI,KAAK;;;AAInB,eAAe,oBAAoB,KAAiB;CAClD,MAAM,EAAC,UAAU,aACf,MAAM,GAA0C;;;KAIhD;AAEF,KAAI,aAAa,UACf,OAAM,IAAI,MACR,uEAAuE,SAAS,GACjF;AAEH,KAAI,UAAA,KACF,OAAM,IAAI,MACR,sDAAsD,QAAQ,IAC/D;AAEH,QAAO;;AAGT,eAAe,sBACb,IACA,KACA,OACA,WAAW,MACwB;CACnC,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,IAAG,OAAO,oCAAoC,SAAS,GAAG,OAAO;AAEjE,OAAM,kBAAkB,IAAI,KAAK,MAAM;CACvC,MAAM,EAAC,iBAAgB,MAAM,uBAAuB,KAAK,MAAM;AAE/D,KAAI,UAAU;EACZ,IAAI,QAAQ;EACZ,MAAM,0BAA0B,aAAa,QAC3C,MAAK,CAAC,EAAE,WAAW,IAAI,CACxB;EACD,MAAM,SAAS,MAAM,GAAG;4DACgC,IAAI,aAAa,CAAC;QACtE,QAAQ;AACZ,MAAI,OAAO,WAAW,aAAa,OACjC,IAAG,OACD,iCAAiC,aAAa,kBACxC,OAAO,MAAM,CAAC,cACrB;WAED,CAAC,OAAO,IAAI,IAAI,MAAM,aAAa,EAAE,IAAI,IAAI,wBAAwB,CAAC,CAEtE,IAAG,OACD,2BAA2B,MAAM,aAAa,sCAC3B,wBAAwB,cAC5C;MAED,SAAQ;AAEV,MAAI,CAAC,OAAO;AACV,SAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,SAAS,CAAC;AACxD,UAAO,sBAAsB,IAAI,KAAK,OAAO,MAAM;;;AAGvD,QAAO,EAAC,cAAa;;AAGvB,SAAS,sBACP,IACA,IACA,UACA,YACA,WACiB;CACjB,MAAM,EAAC,SAAQ,eAAe,SAAS;CACvC,MAAM,eAAe,IAAI,gBACvB,IACA,UACA,MACA,KAAA,GACA,WACD;AACD,cAAa,IAAI,GAAG;AAEpB,IAAG,OAAO,WAAW,WAAW,mBAAmB,UAAU,SAAS;AAEtE,KAAI,SAAS,QAAQ,SAAS,KAAK,GAAG,GACpC,IAAG,OACD,mUAMD;AAEH,QAAO;;AAcT,eAAsB,sBACpB,IACA,SACA,UAEA,WAAW,OACe;CAC1B,MAAM,CAAC,QAAQ,WACX,MAAM,QAAQ,OACJ,4BAA4B,SAAS,+BAC9C,GACD,MAAM,QAAQ,OACJ,4BAA4B,SAAS,oBAC9C;AACL,IAAG,OAAO,4BAA4B,YAAY,KAAK;AACvD,QAAO;;AAGT,SAAS,iBACP,IACA,QACA,gBACA;CAGA,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,GAAG,CAAC;AAChE,MAAK,MAAM,KAAK,QAAQ;AACtB,KAAG,KAAK,yBAAyB,kBAAkB,GAAG,eAAe,CAAC,CAAC;EACvE,MAAM,YAAY,cAAc,EAAE;AAClC,OAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,EAAE,QAAQ,CACxD,gBAAe,OAAO,WAAW,SAAS,QAAQ;;;AAKxD,SAAS,kBAAkB,IAAc,SAAsB;AAC7D,MAAK,MAAM,SAAS,QAClB,IAAG,KAAK,yBAAyB,uBAAuB,MAAM,CAAC,CAAC;;AAUpE,IAAM,KAAK,OAAO;AAClB,IAAM,oBAAoB;AAC1B,IAAM,0BAA0B,IAAI;AAQpC,SAAgB,uBACd,OACA,MACoB;CACpB,MAAM,mBAAmB,OAAO,OAAO,MAAM,aAAa,CACvD,KAAK,EAAC,gBAAe,UAAU,CAC/B,QAAO,MAAK,CAAC,CAAC,EAAE;CACnB,MAAM,QACJ,iBAAiB,WAAW,IACxB,KACQ,SAAS,iBAAiB,KAAK,OAAO;CACpD,MAAM,YAAoB,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG;CACxE,MAAM,aAAa,IAAI,KAAK,KAAI,QAAO,+BAA+B,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;AAMnG,QALc;EACZ,QAAgB,UAAU,KAAK,IAAI,GAAG,CAAC,KAAK,IAAI,CAAC,GAAG;EACpD,cAAsB,kCAAkC;EACxD,eAAuB,UAAU,WAAW,mBAAmB;EAChE;;AASH,eAAe,wBACb,IACA,KACA,MACwB;CACxB,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,QAAQ,cAAc,KAAK;CACjC,MAAM,UAAU,OAAO,KAAK,KAAK,QAAQ;CACzC,MAAM,QAAQ,uBAAuB,MAAM,QAAQ;CACnD,MAAM,aAAa,IAChB,OAA8B,MAAM,aAAa,CACjD,SAAS;CACZ,MAAM,cAAc,IACjB,OAA+B,MAAM,cAAc,CACnD,SAAS;CAEZ,MAAM,QAAuB;EAC3B;EACA,QAAQ;GACN;GACA;GACA,MAAM;GACN,WAAW,QAAQ,MAAM,YAAY,GAAG,UAAU;GAClD,YAAY,QAAQ,MAAM,aAAa,GAAG,WAAW;GACtD;EACF;CACD,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,IAAG,OAAO,uCAAuC,MAAM,IAAI,QAAQ,OAAO,EACxE,OAAO,MAAM,QACd,CAAC;AACF,QAAO;;AAGT,SAAS,KACP,IACA,EAAC,MAAM,OAAO,UACd,UACA,MACA,IACA,UACA;AACA,KAAI,SACF,QAAO,SAAS,IAAI,OAAO,QAAQ,UAAU,MAAM,GAAG;AAExD,QAAO,WAAW,IAAI,OAAO,QAAQ,MAAM,GAAG;;AAGhD,eAAe,WACb,IACA,OACA,QACA,MACA,IACA;CACA,MAAM,QAAQ,YAAY,KAAK;CAC/B,IAAI,YAAY;CAEhB,MAAM,YAAY,cAAc,MAAM;CACtC,MAAM,iBAAiB,OAAO,QAAQ,MAAM,QAAQ;CAEpD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,EAAE;CAClD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,UAAU,KAAK;CAC/D,MAAM,mBAAmB,YAAY,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI;CAE9D,MAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,EAAE,CAAC,MAAM;CACzE,MAAM,YAAoB;mBACT,UAAU,KAAK,iBAAiB,WAAW;CAC5D,MAAM,aAAa,GAAG,QAAQ,UAAU;CACxC,MAAM,kBAAkB,GAAG,QACzB,YAAY,IAAI,YAAY,OAAA,GAA6B,CAC1D;CAGD,MAAM,mBAAmB,OAAO,OAAO,MAAM,aAAa,CACvD,KAAK,EAAC,gBAAe,UAAU,CAC/B,QAAO,MAAK,CAAC,CAAC,EAAE;CACnB,MAAM,QACJ,iBAAiB,WAAW,IACxB,KACQ,SAAS,iBAAiB,KAAK,OAAO;CACpD,MAAM,YAAoB,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG;CAIxE,MAAM,SAAiB,UAHD,eAAe,KAAK,CAAC,MAAM,UAC/C,iBAAiB,KAAK,GAAG,GAAG,KAAK,GAAG,GAAG,GAAG,KAAK,CAAC,QACjD,CAC8C,KAAK,IAAI,CAAC,GAAG;CAE5D,MAAM,WAAW,eAAe,KAAK,GAAG,UACtC,iBAAiB,KAAK,GAAG,kBAAkB,KAAK,GAAG,gBACpD;CAED,MAAM,eAAe,YAAY;CACjC,MAAM,iBAAiB,eAAA;CAEvB,MAAM,gBAAiC,MAAM,KAAK,EAChD,QAAQ,oBAAoB,cAC7B,CAAC;CACF,IAAI,cAAc;CAClB,IAAI,cAAc;CAElB,SAAS,QAAQ;EACf,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,cAAc;EACpB,MAAM,cAAc;EAEpB,IAAI,IAAI;AACR,SAAO,cAAA,IAAiC,eAAA,GACtC,iBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,eAAgB,CAAC;AAEpE,SAAO,cAAc,GAAG,cACtB,YAAW,IAAI,cAAc,MAAM,GAAI,KAAK,aAAc,CAAC;EAE7D,MAAM,gBAAgB,cAAc;AACpC,OAAK,IAAI,IAAI,GAAG,IAAI,eAAe,IACjC,eAAc,KAAK,KAAA;AAErB,gBAAc;AACd,SAAO,QAAQ;EAEf,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,eAAa;AACb,KAAG,QACD,WAAW,YAAY,GAAG,UAAU,SAAS,YAAY,aAAa,QAAQ,QAAQ,EAAE,CAAC,KAC1F;;CAGH,MAAM,eAAe,IAAI,kBAAkB;CAC3C,IAAI,MAAM;AAEV,IAAG,OAAO,kCAAkC,UAAU,IAAI,OAAO;AAEjE,OAAM,WACJ,MAAM,KACH,OAAO,SAAS,OAAO,kCAAkC,CACzD,UAAU,EACb,IAAI,SAAS;EACX,eAAe;EAEf,MACE,OACA,WACA,UACA;AACA,OAAI;AACF,SAAK,MAAM,YAAY,aAAa,MAAM,MAAM,EAAE;AAChD,oBAAe,aAAa,OAAO,IAAI,SAAS;AAChD,mBAAc,cAAc,eAAe,OACzC,aAAa,OAAO,OAAO,SAAS,KAAK,SAAS;AAEpD,SAAI,EAAE,QAAQ,SAAS,QAAQ;AAC7B,YAAM;AACN,UACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,wBAEf,QAAO;;;AAIb,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAI3D,QAAQ,aAAsC;AAC5C,OAAI;AACF,WAAO;AACP,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAG5D,CAAC,CACH;CAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,IAAG,OACD,oBAAoB,OAAO,KAAK,aAAa,UAAU,WAC1C,UAAU,QAAQ,EAAE,CAAC,eAAe,QAAQ,QAAQ,EAAE,CAAC,OACrE;AACD,QAAO;EAAC,MAAM,OAAO;EAAM;EAAU;;AAGvC,eAAe,SACb,IACA,OACA,QACA,UACA,MACA,IACA;CACA,MAAM,QAAQ,YAAY,KAAK;CAC/B,IAAI,YAAY;CAEhB,MAAM,YAAY,cAAc,MAAM;CACtC,MAAM,iBAAiB,OAAO,QAAQ,MAAM,QAAQ;CAEpD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,EAAE;CAClD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,UAAU,KAAK;CAC/D,MAAM,mBAAmB,YAAY,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI;CAE9D,MAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,EAAE,CAAC,MAAM;CACzE,MAAM,YAAoB;mBACT,UAAU,KAAK,iBAAiB,WAAW;CAC5D,MAAM,aAAa,GAAG,QAAQ,UAAU;CACxC,MAAM,kBAAkB,GAAG,QACzB,YAAY,IAAI,YAAY,OAAA,GAA6B,CAC1D;CAED,MAAM,EAAC,WAAU,uBAAuB,OAAO,YAAY;CAC3D,MAAM,eAAe,YAAY;CACjC,MAAM,iBAAiB,eAAA;CAEvB,MAAM,gBAAiC,MAAM,KAAK,EAChD,QAAQ,oBAAoB,cAC7B,CAAC;CACF,IAAI,cAAc;CAClB,IAAI,cAAc;CAElB,SAAS,QAAQ;EACf,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,cAAc;EACpB,MAAM,cAAc;EAEpB,IAAI,IAAI;AACR,SAAO,cAAA,IAAiC,eAAA,GACtC,iBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,eAAgB,CAAC;AAEpE,SAAO,cAAc,GAAG,cACtB,YAAW,IAAI,cAAc,MAAM,GAAI,KAAK,aAAc,CAAC;EAE7D,MAAM,gBAAgB,cAAc;AACpC,OAAK,IAAI,IAAI,GAAG,IAAI,eAAe,IACjC,eAAc,KAAK,KAAA;AAErB,gBAAc;AACd,SAAO,QAAQ;EAEf,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,eAAa;AACb,KAAG,QACD,WAAW,YAAY,GAAG,UAAU,SAAS,YAAY,aAAa,QAAQ,QAAQ,EAAE,CAAC,KAC1F;;AAGH,IAAG,OAAO,gCAAgC,UAAU,IAAI,OAAO;CAC/D,MAAM,YAAY,MAAM,eAAe,UAAU,EAAC,oBAAoB,MAAK,CAAC;CAC5E,MAAM,UAAU,YAAY,KAAI,MAAK;EACnC,MAAM,UAAU,UAAU,cAAc,EAAE,QAAQ;AAClD,UAAQ,QACN,UACE,QAAQ,IAAI,EACZ,EAAE,UAAA,IAEH;GACH;CAEF,MAAM,YAAY,IAAI,WAAW;CACjC,IAAI,MAAM;AAEV,OAAM,WACJ,MAAM,KAAK,OAAO,SAAS,OAAO,aAAa,CAAC,UAAU,EAC1D,IAAI,SAAS;EACX,eAAe;EAEf,MACE,OACA,WACA,UACA;AACA,OAAI;AACF,SAAK,MAAM,QAAQ,UAAU,MAAM,MAAM,EAAE;AACzC,oBAAe,SAAS,OAAO,IAAI,KAAK;AACxC,mBAAc,cAAc,eAAe,OACzC,SAAS,OAAO,OAAO,QAAQ,KAAK,KAAK;AAE3C,SAAI,EAAE,QAAQ,QAAQ,QAAQ;AAC5B,YAAM;AACN,UACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,wBAEf,QAAO;;;AAIb,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAI3D,QAAQ,aAAsC;AAC5C,OAAI;AACF,WAAO;AACP,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAG5D,CAAC,CACH;CAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,IAAG,OACD,oBAAoB,OAAO,KAAK,aAAa,UAAU,WAC1C,UAAU,QAAQ,EAAE,CAAC,eAAe,QAAQ,QAAQ,EAAE,CAAC,OACrE;AACD,QAAO;EAAC,MAAM,OAAO;EAAM;EAAU"}

package/out/zero-cache/src/services/change-source/pg/schema/shard.js CHANGED Viewed

@@ -116,7 +116,7 @@ var SHARD_CONFIG_TABLE = "shardConfig";
 function shardSetup(shardConfig, metadataPublication) {
 	const app = id(appSchema(shardConfig));
 	const shard = id(upstreamSchema(shardConfig));
-	const pubs = [...shardConfig.publications].sort();
+	const pubs = shardConfig.publications.toSorted();
 	assert(pubs.includes(metadataPublication), () => `Publications must include ${metadataPublication}`);
 	return `
   CREATE SCHEMA IF NOT EXISTS ${shard};
@@ -257,7 +257,7 @@ async function setupTriggers(lc, tx, shard) {
 }
 function validatePublications(lc, published) {
 	published.publications.forEach((pub) => {
-		if (!pub.pubinsert || !pub.pubtruncate || !pub.pubdelete || !pub.pubtruncate) throw new Error(`PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`);
+		if (!pub.pubinsert || !pub.pubupdate || !pub.pubdelete || !pub.pubtruncate) throw new Error(`PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`);
 	});
 	published.tables.forEach((table) => validate(lc, table));
 }

package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"shard.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {\n jsonObjectSchema,\n stringify,\n type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {createEventTriggerStatements} from './ddl.ts';\nimport {\n getPublicationInfo,\n publishedSchema,\n type PublicationInfo,\n type PublishedSchema,\n} from './published.ts';\nimport {validate} from './validation.ts';\n\n/*\n PostgreSQL unquoted identifiers must start with a letter or underscore\n * and contain only letters, digits, and underscores.\n /\nconst VALID_PUBLICATION_NAME = /^[a-zA-Z_][a-zA-Z0-9_]$/;\n\n/*\n Validates that a publication name is a valid PostgreSQL identifier.\n * This provides defense-in-depth against SQL injection when publication\n * names are used in replication commands.\n /\nexport function validatePublicationName(name: string): void {\n if (!VALID_PUBLICATION_NAME.test(name)) {\n throw new Error(\n `Invalid publication name \"${name}\". Publication names must start with a letter or underscore ` +\n `and contain only letters, digits, and underscores.`,\n );\n }\n if (name.length > 63) {\n throw new Error(\n `Publication name \"${name}\" exceeds PostgreSQL's 63-character identifier limit.`,\n );\n }\n}\n\nexport function internalPublicationPrefix({appID}: AppID) {\n return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n const {appID, shardNum} = check(shard);\n return `${appID}_${shardNum}_`;\n}\n\n/\n An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n /\nexport function replicationSlotExpression(shard: ShardID) {\n // Underscores have a special meaning in LIKE values\n // so they have to be escaped.\n return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nexport function newReplicationSlot(shard: ShardID) {\n return replicationSlotPrefix(shard) + Date.now();\n}\n\nfunction defaultPublicationName(appID: string, shardID: string \| number) {\n return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n appID: string,\n shardID: string \| number,\n) {\n return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n const app = id(appSchema(appID));\n\n return /sql/ `\n CREATE SCHEMA IF NOT EXISTS ${app};\n\n CREATE TABLE IF NOT EXISTS ${app}.permissions (\n \"permissions\" JSONB,\n \"hash\" TEXT,\n\n -- Ensure that there is only a single row in the table.\n -- Application code can be agnostic to this column, and\n -- simply invoke UPDATE statements on the version columns.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n RETURNS TRIGGER AS $$\n BEGIN\n NEW.hash = md5(NEW.permissions::text);\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n\n CREATE OR REPLACE TRIGGER on_set_permissions \n BEFORE INSERT OR UPDATE ON ${app}.permissions\n FOR EACH ROW\n EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n return /sql/ `\n CREATE TABLE ${schema}.\"clients\" (\n \"clientGroupID\" TEXT NOT NULL,\n \"clientID\" TEXT NOT NULL,\n \"lastMutationID\" BIGINT NOT NULL,\n \"userID\" TEXT,\n PRIMARY KEY(\"clientGroupID\", \"clientID\")\n );`;\n}\n\n/\n Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n \n The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n /\nexport function getMutationsTableDefinition(schema: string) {\n return /sql/ `\n CREATE TABLE ${schema}.\"mutations\" (\n \"clientGroupID\" TEXT NOT NULL,\n \"clientID\" TEXT NOT NULL,\n \"mutationID\" BIGINT NOT NULL,\n \"result\" JSON NOT NULL,\n PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n shardConfig: ShardConfig,\n metadataPublication: string,\n): string {\n const app = id(appSchema(shardConfig));\n const shard = id(upstreamSchema(shardConfig));\n\n const pubs = [...shardConfig.publications].sort();\n assert(\n pubs.includes(metadataPublication),\n () => `Publications must include ${metadataPublication}`,\n );\n\n return /sql/ `\n CREATE SCHEMA IF NOT EXISTS ${shard};\n\n ${getClientsTableDefinition(shard)}\n ${getMutationsTableDefinition(shard)}\n\n DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n CREATE PUBLICATION ${id(metadataPublication)}\n FOR TABLE ${app}.\"permissions\", TABLE ${shard}.\"clients\", ${shard}.\"mutations\";\n\n CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n \"publications\" TEXT[] NOT NULL,\n \"ddlDetection\" BOOL NOT NULL,\n\n -- Ensure that there is only a single row in the table.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n \"publications\",\n \"ddlDetection\" \n ) VALUES (\n ARRAY[${literal(pubs)}], \n false -- set in SAVEPOINT with triggerSetup() statements\n );\n\n CREATE TABLE ${shard}.replicas (\n \"slot\" TEXT PRIMARY KEY,\n \"version\" TEXT NOT NULL,\n \"initialSchema\" JSON NOT NULL,\n \"initialSyncContext\" JSON,\n \"subscriberContext\" JSON\n );\n `;\n}\n\nexport function dropShard(appID: string, shardID: string \| number): string {\n const schema = `${appID}_${shardID}`;\n const metadataPublication = metadataPublicationName(appID, shardID);\n const defaultPublication = defaultPublicationName(appID, shardID);\n\n // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n // so PUBLICATIONs must be dropped explicitly.\n return /sql/ `\n DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n `;\n}\n\nconst internalShardConfigSchema = v.object({\n publications: v.array(v.string()),\n ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaSchema = internalShardConfigSchema.extend({\n slot: v.string(),\n version: v.string(),\n initialSchema: publishedSchema,\n initialSyncContext: jsonObjectSchema.nullable(),\n subscriberContext: jsonObjectSchema.nullable(),\n});\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n const schema = id(upstreamSchema(shard));\n return (\n createEventTriggerStatements(shard) +\n /sql/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n );\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function addReplica(\n sql: PostgresDB,\n shard: ShardID,\n slot: string,\n replicaVersion: string,\n {tables, indexes}: PublishedSchema,\n initialSyncContext: JSONObject,\n) {\n const schema = upstreamSchema(shard);\n const synced: PublishedSchema = {tables, indexes};\n await sql`\n INSERT INTO ${sql(schema)}.replicas\n (\"slot\", \"version\", \"initialSchema\", \"initialSyncContext\")\n VALUES (${slot}, ${replicaVersion}, ${synced}, ${initialSyncContext})`;\n}\n\nexport async function getReplicaAtVersion(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n replicaVersion: string,\n context?: JSONObject,\n): Promise<Replica \| null> {\n const schema = sql(upstreamSchema(shard));\n const result = await sql`\n SELECT FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n WHERE version = ${replicaVersion};\n `;\n if (result.length === 0) {\n // log out all the replicas and the joined shardConfig\n const allReplicas = await sql`\n SELECT slot, version, \"initialSyncContext\", \"subscriberContext\" \n FROM ${schema}.replicas`;\n lc.info?.(\n `Replica ${replicaVersion} ` +\n (context ? `(context: ${stringify(context)}) ` : '') +\n `not found in: ${stringify(allReplicas)}`,\n );\n return null;\n }\n return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getInternalShardConfig(\n sql: PostgresDB,\n shard: ShardID,\n): Promise<InternalShardConfig> {\n const result = await sql`\n SELECT \"publications\", \"ddlDetection\"\n FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n `;\n assert(\n result.length === 1,\n () => `Expected exactly one shardConfig row, got ${result.length}`,\n );\n return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/*\n Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n lc: LogContext,\n sql: PostgresTransaction,\n requested: ShardConfig,\n) {\n const {publications} = requested;\n // Validate requested publications.\n for (const pub of publications) {\n validatePublicationName(pub);\n if (pub.startsWith('_')) {\n throw new Error(\n `Publication names starting with \"_\" are reserved for internal use.\\n` +\n `Please use a different name for publication \"${pub}\".`,\n );\n }\n }\n const allPublications: string[] = [];\n\n // Setup application publications.\n if (publications.length) {\n const results = await sql<{pubname: string}[]>`\n SELECT pubname from pg_publication WHERE pubname IN ${sql(\n publications,\n )}`.values();\n\n if (results.length !== publications.length) {\n throw new Error(\n `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n );\n }\n allPublications.push(...publications);\n } else {\n const defaultPublication = defaultPublicationName(\n requested.appID,\n requested.shardNum,\n );\n await sql`\n DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n await sql`\n CREATE PUBLICATION ${sql(defaultPublication)} \n FOR TABLES IN SCHEMA public\n WITH (publish_via_partition_root = true)`;\n allPublications.push(defaultPublication);\n }\n\n const metadataPublication = metadataPublicationName(\n requested.appID,\n requested.shardNum,\n );\n allPublications.push(metadataPublication);\n\n const shard = {...requested, publications: allPublications};\n\n // Setup the global tables and shard tables / publications.\n await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n const pubs = await getPublicationInfo(sql, allPublications);\n await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n await setupTriggers(lc, sql, shard);\n}\n\nexport async function setupTriggers(\n lc: LogContext,\n tx: PostgresTransaction,\n shard: ShardConfig,\n) {\n try {\n await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n } catch (e) {\n if (\n !(\n e instanceof postgres.PostgresError &&\n e.code === PG_INSUFFICIENT_PRIVILEGE\n )\n ) {\n throw e;\n }\n // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n lc.warn?.(\n `Unable to create event triggers for schema change detection:\\n\\n` +\n `\"${e.hint ?? e.message}\"\\n\\n` +\n `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n `requiring the replica to be reset (manually or with --auto-reset).`,\n );\n }\n}\n\nexport function validatePublications(\n lc: LogContext,\n published: PublicationInfo,\n) {\n // Verify that all publications export the proper events.\n published.publications.forEach(pub => {\n if (\n !pub.pubinsert \|\|\n !pub.pubtruncate \|\|\n !pub.pubdelete \|\|\n !pub.pubtruncate\n ) {\n // TODO: Make APIError?\n throw new Error(\n `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n );\n }\n });\n\n published.tables.forEach(table => validate(lc, table));\n}\n\ntype ReplicaIdentities = {\n apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n pubs: PublishedSchema,\n): ReplicaIdentities \| undefined {\n const replicaIdentities: {\n schema: string;\n tableName: string;\n indexName: string;\n }[] = [];\n for (const table of pubs.tables) {\n if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n // - UNIQUE\n // - NOT NULL columns\n // - not deferrable (i.e. isImmediate)\n // - not partial (are already filtered out)\n //\n // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n const {schema, name: tableName} = table;\n for (const {columns, name: indexName} of pubs.indexes.filter(\n idx =>\n idx.schema === schema &&\n idx.tableName === tableName &&\n idx.unique &&\n idx.isImmediate,\n )) {\n if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n continue; // Only indexes with all NOT NULL columns are suitable.\n }\n replicaIdentities.push({schema, tableName, indexName});\n break;\n }\n }\n }\n\n if (replicaIdentities.length === 0) {\n return undefined;\n }\n return {\n apply: async (lc: LogContext, sql: PostgresDB) => {\n for (const {schema, tableName, indexName} of replicaIdentities) {\n lc.info?.(\n `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n );\n await sql`\n ALTER TABLE ${sql(schema)}.${sql(tableName)} \n REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n }\n },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA6BA,IAAM,yBAAyB;;;;;;AAO/B,SAAgB,wBAAwB,MAAoB;AAC1D,KAAI,CAAC,uBAAuB,KAAK,KAAK,CACpC,OAAM,IAAI,MACR,6BAA6B,KAAK,gHAEnC;AAEH,KAAI,KAAK,SAAS,GAChB,OAAM,IAAI,MACR,qBAAqB,KAAK,uDAC3B;;AAIL,SAAgB,0BAA0B,EAAC,SAAe;AACxD,QAAO,IAAI,MAAM;;AAGnB,SAAgB,sBAAsB,EAAC,OAAO,YAAoB;AAChE,QAAO,GAAG,MAAM,GAAG;;AAGrB,SAAgB,sBAAsB,OAAgB;CACpD,MAAM,EAAC,OAAO,aAAY,MAAM,MAAM;AACtC,QAAO,GAAG,MAAM,GAAG,SAAS;;;;;;AAO9B,SAAgB,0BAA0B,OAAgB;AAGxD,QAAO,GAAG,sBAAsB,MAAM,CAAC,GAAG,WAAW,KAAK,MAAM;;AAGlE,SAAgB,mBAAmB,OAAgB;AACjD,QAAO,sBAAsB,MAAM,GAAG,KAAK,KAAK;;AAGlD,SAAS,uBAAuB,OAAe,SAA0B;AACvE,QAAO,IAAI,MAAM,UAAU;;AAG7B,SAAgB,wBACd,OACA,SACA;AACA,QAAO,IAAI,MAAM,YAAY;;AAI/B,SAAS,YAAY,OAAsB;CACzC,MAAM,MAAM,GAAG,UAAU,MAAM,CAAC;AAEhC,QAAe;gCACe,IAAI;;+BAEL,IAAI;;;;;;;;;;+BAUJ,IAAI;;;;;;;;;iCASF,IAAI;;uBAEd,IAAI;;gBAEX,IAAI;;;AAIpB,eAAsB,mBAAmB,IAAgB,OAAc;AACrE,OAAM,GAAG,OAAO,YAAY,MAAM,CAAC;;AAGrC,SAAgB,0BAA0B,QAAgB;AACxD,QAAe;iBACA,OAAO;;;;;;;;;;;;;;;;AAiBxB,SAAgB,4BAA4B,QAAgB;AAC1D,QAAe;iBACA,OAAO;;;;;;;;AASxB,IAAa,qBAAqB;AAElC,SAAgB,WACd,aACA,qBACQ;CACR,MAAM,MAAM,GAAG,UAAU,YAAY,CAAC;CACtC,MAAM,QAAQ,GAAG,eAAe,YAAY,CAAC;CAE7C,MAAM,OAAO,CAAC,GAAG,YAAY,aAAa,CAAC,MAAM;AACjD,QACE,KAAK,SAAS,oBAAoB,QAC5B,6BAA6B,sBACpC;AAED,QAAe;gCACe,MAAM;;IAElC,0BAA0B,MAAM,CAAC;IACjC,4BAA4B,MAAM,CAAC;;+BAER,GAAG,oBAAoB,CAAC;uBAChC,GAAG,oBAAoB,CAAC;gBAC/B,IAAI,wBAAwB,MAAM,cAAc,MAAM;;iBAErD,MAAM,IAAI,mBAAmB;;;;;;;;gBAQ9B,MAAM,IAAI,mBAAmB;;;;cAI/B,QAAQ,KAAK,CAAC;;;;iBAIX,MAAM;;;;;;;;;AAUvB,SAAgB,UAAU,OAAe,SAAkC;CACzE,MAAM,SAAS,GAAG,MAAM,GAAG;CAC3B,MAAM,sBAAsB,wBAAwB,OAAO,QAAQ;AAKnE,QAAe;iCACgB,GALJ,uBAAuB,OAAO,QAAQ,CAKZ,CAAC;iCACvB,GAAG,oBAAoB,CAAC;4BAC7B,GAAG,OAAO,CAAC;;;AAIvC,IAAM,4BAA4B,eAAE,OAAO;CACzC,cAAc,eAAE,MAAM,eAAE,QAAQ,CAAC;CACjC,cAAc,eAAE,SAAS;CAC1B,CAAC;AAIF,IAAM,gBAAgB,0BAA0B,OAAO;CACrD,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ;CACnB,eAAe;CACf,oBAAoB,iBAAiB,UAAU;CAC/C,mBAAmB,iBAAiB,UAAU;CAC/C,CAAC;AAOF,SAAS,aAAa,OAA4B;CAChD,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;AACxC,QACE,6BAA6B,MAAM,GAC3B,UAAU,OAAO;;AAK7B,eAAsB,WACpB,KACA,OACA,MACA,gBACA,EAAC,QAAQ,WACT,oBACA;CACA,MAAM,SAAS,eAAe,MAAM;CACpC,MAAM,SAA0B;EAAC;EAAQ;EAAQ;AACjD,OAAM,GAAG;kBACO,IAAI,OAAO,CAAC;;gBAEd,KAAK,IAAI,eAAe,IAAI,OAAO,IAAI,mBAAmB;;AAG1E,eAAsB,oBACpB,IACA,KACA,OACA,gBACA,SACyB;CACzB,MAAM,SAAS,IAAI,eAAe,MAAM,CAAC;CACzC,MAAM,SAAS,MAAM,GAAG;oBACN,OAAO,iBAAiB,OAAO;wBAC3B,eAAe;;AAErC,KAAI,OAAO,WAAW,GAAG;EAEvB,MAAM,cAAc,MAAM,GAAG;;eAElB,OAAO;AAClB,KAAG,OACD,WAAW,eAAe,MACvB,UAAU,aAAa,UAAU,QAAQ,CAAC,MAAM,MACjD,iBAAiB,UAAU,YAAY,GAC1C;AACD,SAAO;;AAET,QAAO,MAAQ,OAAO,IAAI,eAAe,cAAc;;AAGzD,eAAsB,uBACpB,KACA,OAC8B;CAC9B,MAAM,SAAS,MAAM,GAAG;;aAEb,IAAI,eAAe,MAAM,CAAC,CAAC;;AAEtC,QACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,SAC3D;AACD,QAAO,MAAQ,OAAO,IAAI,2BAA2B,cAAc;;;;;;AAOrE,eAAsB,0BACpB,IACA,KACA,WACA;CACA,MAAM,EAAC,iBAAgB;AAEvB,MAAK,MAAM,OAAO,cAAc;AAC9B,0BAAwB,IAAI;AAC5B,MAAI,IAAI,WAAW,IAAI,CACrB,OAAM,IAAI,MACR,oHACkD,IAAI,IACvD;;CAGL,MAAM,kBAA4B,EAAE;AAGpC,KAAI,aAAa,QAAQ;EACvB,MAAM,UAAU,MAAM,GAAwB;0DACQ,IACpD,aACD,GAAG,QAAQ;AAEZ,MAAI,QAAQ,WAAW,aAAa,OAClC,OAAM,IAAI,MACR,gDAAgD,aAAa,aAAa,QAAQ,MAAM,CAAC,GAC1F;AAEH,kBAAgB,KAAK,GAAG,aAAa;QAChC;EACL,MAAM,qBAAqB,uBACzB,UAAU,OACV,UAAU,SACX;AACD,QAAM,GAAG;mCACsB,IAAI,mBAAmB;AACtD,QAAM,GAAG;2BACc,IAAI,mBAAmB,CAAC;;;AAG/C,kBAAgB,KAAK,mBAAmB;;CAG1C,MAAM,sBAAsB,wBAC1B,UAAU,OACV,UAAU,SACX;AACD,iBAAgB,KAAK,oBAAoB;CAEzC,MAAM,QAAQ;EAAC,GAAG;EAAW,cAAc;EAAgB;AAG3D,OAAM,IAAI,OAAO,YAAY,MAAM,GAAG,WAAW,OAAO,oBAAoB,CAAC;AAG7E,OAAM,6CADO,MAAM,mBAAmB,KAAK,gBAAgB,CACH,EAAE,MAAM,IAAI,IAAI;AAExE,OAAM,cAAc,IAAI,KAAK,MAAM;;AAGrC,eAAsB,cACpB,IACA,IACA,OACA;AACA,KAAI;AACF,QAAM,GAAG,WAAU,QAAO,IAAI,OAAO,aAAa,MAAM,CAAC,CAAC;UACnD,GAAG;AACV,MACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,2BAGb,OAAM;AAGR,KAAG,OACD,oEACM,EAAE,QAAQ,EAAE,QAAQ,6IAG3B;;;AAIL,SAAgB,qBACd,IACA,WACA;AAEA,WAAU,aAAa,SAAQ,QAAO;AACpC,MACE,CAAC,IAAI,aACL,CAAC,IAAI,eACL,CAAC,IAAI,aACL,CAAC,IAAI,YAGL,OAAM,IAAI,MACR,eAAe,IAAI,QAAQ,oDAC5B;GAEH;AAEF,WAAU,OAAO,SAAQ,UAAS,SAAS,IAAI,MAAM,CAAC;;AAOxD,SAAgB,6CACd,MAC+B;CAC/B,MAAM,oBAIA,EAAE;AACR,MAAK,MAAM,SAAS,KAAK,OACvB,KAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAA,KAA6B;EAQlE,MAAM,EAAC,QAAQ,MAAM,cAAa;AAClC,OAAK,MAAM,EAAC,SAAS,MAAM,eAAc,KAAK,QAAQ,QACpD,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI,YACP,EAAE;AACD,OAAI,OAAO,KAAK,QAAQ,CAAC,MAAK,QAAO,CAAC,MAAM,QAAQ,KAAK,QAAQ,CAC/D;AAEF,qBAAkB,KAAK;IAAC;IAAQ;IAAW;IAAU,CAAC;AACtD;;;AAKN,KAAI,kBAAkB,WAAW,EAC/B;AAEF,QAAO,EACL,OAAO,OAAO,IAAgB,QAAoB;AAChD,OAAK,MAAM,EAAC,QAAQ,WAAW,eAAc,mBAAmB;AAC9D,MAAG,OACD,YAAY,UAAU,iCAAiC,UAAU,GAClE;AACD,SAAM,GAAG;sBACK,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,CAAC;yCACX,IAAI,UAAU;;IAGpD"}
1	+ {"version":3,"file":"shard.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {\n jsonObjectSchema,\n stringify,\n type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {createEventTriggerStatements} from './ddl.ts';\nimport {\n getPublicationInfo,\n publishedSchema,\n type PublicationInfo,\n type PublishedSchema,\n} from './published.ts';\nimport {validate} from './validation.ts';\n\n/*\n PostgreSQL unquoted identifiers must start with a letter or underscore\n * and contain only letters, digits, and underscores.\n /\nconst VALID_PUBLICATION_NAME = /^[a-zA-Z_][a-zA-Z0-9_]$/;\n\n/*\n Validates that a publication name is a valid PostgreSQL identifier.\n * This provides defense-in-depth against SQL injection when publication\n * names are used in replication commands.\n /\nexport function validatePublicationName(name: string): void {\n if (!VALID_PUBLICATION_NAME.test(name)) {\n throw new Error(\n `Invalid publication name \"${name}\". Publication names must start with a letter or underscore ` +\n `and contain only letters, digits, and underscores.`,\n );\n }\n if (name.length > 63) {\n throw new Error(\n `Publication name \"${name}\" exceeds PostgreSQL's 63-character identifier limit.`,\n );\n }\n}\n\nexport function internalPublicationPrefix({appID}: AppID) {\n return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n const {appID, shardNum} = check(shard);\n return `${appID}_${shardNum}_`;\n}\n\n/\n An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n /\nexport function replicationSlotExpression(shard: ShardID) {\n // Underscores have a special meaning in LIKE values\n // so they have to be escaped.\n return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nexport function newReplicationSlot(shard: ShardID) {\n return replicationSlotPrefix(shard) + Date.now();\n}\n\nfunction defaultPublicationName(appID: string, shardID: string \| number) {\n return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n appID: string,\n shardID: string \| number,\n) {\n return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n const app = id(appSchema(appID));\n\n return /sql/ `\n CREATE SCHEMA IF NOT EXISTS ${app};\n\n CREATE TABLE IF NOT EXISTS ${app}.permissions (\n \"permissions\" JSONB,\n \"hash\" TEXT,\n\n -- Ensure that there is only a single row in the table.\n -- Application code can be agnostic to this column, and\n -- simply invoke UPDATE statements on the version columns.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n RETURNS TRIGGER AS $$\n BEGIN\n NEW.hash = md5(NEW.permissions::text);\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n\n CREATE OR REPLACE TRIGGER on_set_permissions \n BEFORE INSERT OR UPDATE ON ${app}.permissions\n FOR EACH ROW\n EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n return /sql/ `\n CREATE TABLE ${schema}.\"clients\" (\n \"clientGroupID\" TEXT NOT NULL,\n \"clientID\" TEXT NOT NULL,\n \"lastMutationID\" BIGINT NOT NULL,\n \"userID\" TEXT,\n PRIMARY KEY(\"clientGroupID\", \"clientID\")\n );`;\n}\n\n/\n Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n \n The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n /\nexport function getMutationsTableDefinition(schema: string) {\n return /sql/ `\n CREATE TABLE ${schema}.\"mutations\" (\n \"clientGroupID\" TEXT NOT NULL,\n \"clientID\" TEXT NOT NULL,\n \"mutationID\" BIGINT NOT NULL,\n \"result\" JSON NOT NULL,\n PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n shardConfig: ShardConfig,\n metadataPublication: string,\n): string {\n const app = id(appSchema(shardConfig));\n const shard = id(upstreamSchema(shardConfig));\n\n const pubs = shardConfig.publications.toSorted();\n assert(\n pubs.includes(metadataPublication),\n () => `Publications must include ${metadataPublication}`,\n );\n\n return /sql/ `\n CREATE SCHEMA IF NOT EXISTS ${shard};\n\n ${getClientsTableDefinition(shard)}\n ${getMutationsTableDefinition(shard)}\n\n DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n CREATE PUBLICATION ${id(metadataPublication)}\n FOR TABLE ${app}.\"permissions\", TABLE ${shard}.\"clients\", ${shard}.\"mutations\";\n\n CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n \"publications\" TEXT[] NOT NULL,\n \"ddlDetection\" BOOL NOT NULL,\n\n -- Ensure that there is only a single row in the table.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n \"publications\",\n \"ddlDetection\" \n ) VALUES (\n ARRAY[${literal(pubs)}], \n false -- set in SAVEPOINT with triggerSetup() statements\n );\n\n CREATE TABLE ${shard}.replicas (\n \"slot\" TEXT PRIMARY KEY,\n \"version\" TEXT NOT NULL,\n \"initialSchema\" JSON NOT NULL,\n \"initialSyncContext\" JSON,\n \"subscriberContext\" JSON\n );\n `;\n}\n\nexport function dropShard(appID: string, shardID: string \| number): string {\n const schema = `${appID}_${shardID}`;\n const metadataPublication = metadataPublicationName(appID, shardID);\n const defaultPublication = defaultPublicationName(appID, shardID);\n\n // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n // so PUBLICATIONs must be dropped explicitly.\n return /sql/ `\n DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n `;\n}\n\nconst internalShardConfigSchema = v.object({\n publications: v.array(v.string()),\n ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaSchema = internalShardConfigSchema.extend({\n slot: v.string(),\n version: v.string(),\n initialSchema: publishedSchema,\n initialSyncContext: jsonObjectSchema.nullable(),\n subscriberContext: jsonObjectSchema.nullable(),\n});\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n const schema = id(upstreamSchema(shard));\n return (\n createEventTriggerStatements(shard) +\n /sql/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n );\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function addReplica(\n sql: PostgresDB,\n shard: ShardID,\n slot: string,\n replicaVersion: string,\n {tables, indexes}: PublishedSchema,\n initialSyncContext: JSONObject,\n) {\n const schema = upstreamSchema(shard);\n const synced: PublishedSchema = {tables, indexes};\n await sql`\n INSERT INTO ${sql(schema)}.replicas\n (\"slot\", \"version\", \"initialSchema\", \"initialSyncContext\")\n VALUES (${slot}, ${replicaVersion}, ${synced}, ${initialSyncContext})`;\n}\n\nexport async function getReplicaAtVersion(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n replicaVersion: string,\n context?: JSONObject,\n): Promise<Replica \| null> {\n const schema = sql(upstreamSchema(shard));\n const result = await sql`\n SELECT FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n WHERE version = ${replicaVersion};\n `;\n if (result.length === 0) {\n // log out all the replicas and the joined shardConfig\n const allReplicas = await sql`\n SELECT slot, version, \"initialSyncContext\", \"subscriberContext\" \n FROM ${schema}.replicas`;\n lc.info?.(\n `Replica ${replicaVersion} ` +\n (context ? `(context: ${stringify(context)}) ` : '') +\n `not found in: ${stringify(allReplicas)}`,\n );\n return null;\n }\n return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getInternalShardConfig(\n sql: PostgresDB,\n shard: ShardID,\n): Promise<InternalShardConfig> {\n const result = await sql`\n SELECT \"publications\", \"ddlDetection\"\n FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n `;\n assert(\n result.length === 1,\n () => `Expected exactly one shardConfig row, got ${result.length}`,\n );\n return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/*\n Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n lc: LogContext,\n sql: PostgresTransaction,\n requested: ShardConfig,\n) {\n const {publications} = requested;\n // Validate requested publications.\n for (const pub of publications) {\n validatePublicationName(pub);\n if (pub.startsWith('_')) {\n throw new Error(\n `Publication names starting with \"_\" are reserved for internal use.\\n` +\n `Please use a different name for publication \"${pub}\".`,\n );\n }\n }\n const allPublications: string[] = [];\n\n // Setup application publications.\n if (publications.length) {\n const results = await sql<{pubname: string}[]>`\n SELECT pubname from pg_publication WHERE pubname IN ${sql(\n publications,\n )}`.values();\n\n if (results.length !== publications.length) {\n throw new Error(\n `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n );\n }\n allPublications.push(...publications);\n } else {\n const defaultPublication = defaultPublicationName(\n requested.appID,\n requested.shardNum,\n );\n await sql`\n DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n await sql`\n CREATE PUBLICATION ${sql(defaultPublication)} \n FOR TABLES IN SCHEMA public\n WITH (publish_via_partition_root = true)`;\n allPublications.push(defaultPublication);\n }\n\n const metadataPublication = metadataPublicationName(\n requested.appID,\n requested.shardNum,\n );\n allPublications.push(metadataPublication);\n\n const shard = {...requested, publications: allPublications};\n\n // Setup the global tables and shard tables / publications.\n await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n const pubs = await getPublicationInfo(sql, allPublications);\n await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n await setupTriggers(lc, sql, shard);\n}\n\nexport async function setupTriggers(\n lc: LogContext,\n tx: PostgresTransaction,\n shard: ShardConfig,\n) {\n try {\n await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n } catch (e) {\n if (\n !(\n e instanceof postgres.PostgresError &&\n e.code === PG_INSUFFICIENT_PRIVILEGE\n )\n ) {\n throw e;\n }\n // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n lc.warn?.(\n `Unable to create event triggers for schema change detection:\\n\\n` +\n `\"${e.hint ?? e.message}\"\\n\\n` +\n `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n `requiring the replica to be reset (manually or with --auto-reset).`,\n );\n }\n}\n\nexport function validatePublications(\n lc: LogContext,\n published: PublicationInfo,\n) {\n // Verify that all publications export the proper events.\n published.publications.forEach(pub => {\n if (\n !pub.pubinsert \|\|\n !pub.pubupdate \|\|\n !pub.pubdelete \|\|\n !pub.pubtruncate\n ) {\n // TODO: Make APIError?\n throw new Error(\n `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n );\n }\n });\n\n published.tables.forEach(table => validate(lc, table));\n}\n\ntype ReplicaIdentities = {\n apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n pubs: PublishedSchema,\n): ReplicaIdentities \| undefined {\n const replicaIdentities: {\n schema: string;\n tableName: string;\n indexName: string;\n }[] = [];\n for (const table of pubs.tables) {\n if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n // - UNIQUE\n // - NOT NULL columns\n // - not deferrable (i.e. isImmediate)\n // - not partial (are already filtered out)\n //\n // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n const {schema, name: tableName} = table;\n for (const {columns, name: indexName} of pubs.indexes.filter(\n idx =>\n idx.schema === schema &&\n idx.tableName === tableName &&\n idx.unique &&\n idx.isImmediate,\n )) {\n if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n continue; // Only indexes with all NOT NULL columns are suitable.\n }\n replicaIdentities.push({schema, tableName, indexName});\n break;\n }\n }\n }\n\n if (replicaIdentities.length === 0) {\n return undefined;\n }\n return {\n apply: async (lc: LogContext, sql: PostgresDB) => {\n for (const {schema, tableName, indexName} of replicaIdentities) {\n lc.info?.(\n `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n );\n await sql`\n ALTER TABLE ${sql(schema)}.${sql(tableName)} \n REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n }\n },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA6BA,IAAM,yBAAyB;;;;;;AAO/B,SAAgB,wBAAwB,MAAoB;AAC1D,KAAI,CAAC,uBAAuB,KAAK,KAAK,CACpC,OAAM,IAAI,MACR,6BAA6B,KAAK,gHAEnC;AAEH,KAAI,KAAK,SAAS,GAChB,OAAM,IAAI,MACR,qBAAqB,KAAK,uDAC3B;;AAIL,SAAgB,0BAA0B,EAAC,SAAe;AACxD,QAAO,IAAI,MAAM;;AAGnB,SAAgB,sBAAsB,EAAC,OAAO,YAAoB;AAChE,QAAO,GAAG,MAAM,GAAG;;AAGrB,SAAgB,sBAAsB,OAAgB;CACpD,MAAM,EAAC,OAAO,aAAY,MAAM,MAAM;AACtC,QAAO,GAAG,MAAM,GAAG,SAAS;;;;;;AAO9B,SAAgB,0BAA0B,OAAgB;AAGxD,QAAO,GAAG,sBAAsB,MAAM,CAAC,GAAG,WAAW,KAAK,MAAM;;AAGlE,SAAgB,mBAAmB,OAAgB;AACjD,QAAO,sBAAsB,MAAM,GAAG,KAAK,KAAK;;AAGlD,SAAS,uBAAuB,OAAe,SAA0B;AACvE,QAAO,IAAI,MAAM,UAAU;;AAG7B,SAAgB,wBACd,OACA,SACA;AACA,QAAO,IAAI,MAAM,YAAY;;AAI/B,SAAS,YAAY,OAAsB;CACzC,MAAM,MAAM,GAAG,UAAU,MAAM,CAAC;AAEhC,QAAe;gCACe,IAAI;;+BAEL,IAAI;;;;;;;;;;+BAUJ,IAAI;;;;;;;;;iCASF,IAAI;;uBAEd,IAAI;;gBAEX,IAAI;;;AAIpB,eAAsB,mBAAmB,IAAgB,OAAc;AACrE,OAAM,GAAG,OAAO,YAAY,MAAM,CAAC;;AAGrC,SAAgB,0BAA0B,QAAgB;AACxD,QAAe;iBACA,OAAO;;;;;;;;;;;;;;;;AAiBxB,SAAgB,4BAA4B,QAAgB;AAC1D,QAAe;iBACA,OAAO;;;;;;;;AASxB,IAAa,qBAAqB;AAElC,SAAgB,WACd,aACA,qBACQ;CACR,MAAM,MAAM,GAAG,UAAU,YAAY,CAAC;CACtC,MAAM,QAAQ,GAAG,eAAe,YAAY,CAAC;CAE7C,MAAM,OAAO,YAAY,aAAa,UAAU;AAChD,QACE,KAAK,SAAS,oBAAoB,QAC5B,6BAA6B,sBACpC;AAED,QAAe;gCACe,MAAM;;IAElC,0BAA0B,MAAM,CAAC;IACjC,4BAA4B,MAAM,CAAC;;+BAER,GAAG,oBAAoB,CAAC;uBAChC,GAAG,oBAAoB,CAAC;gBAC/B,IAAI,wBAAwB,MAAM,cAAc,MAAM;;iBAErD,MAAM,IAAI,mBAAmB;;;;;;;;gBAQ9B,MAAM,IAAI,mBAAmB;;;;cAI/B,QAAQ,KAAK,CAAC;;;;iBAIX,MAAM;;;;;;;;;AAUvB,SAAgB,UAAU,OAAe,SAAkC;CACzE,MAAM,SAAS,GAAG,MAAM,GAAG;CAC3B,MAAM,sBAAsB,wBAAwB,OAAO,QAAQ;AAKnE,QAAe;iCACgB,GALJ,uBAAuB,OAAO,QAAQ,CAKZ,CAAC;iCACvB,GAAG,oBAAoB,CAAC;4BAC7B,GAAG,OAAO,CAAC;;;AAIvC,IAAM,4BAA4B,eAAE,OAAO;CACzC,cAAc,eAAE,MAAM,eAAE,QAAQ,CAAC;CACjC,cAAc,eAAE,SAAS;CAC1B,CAAC;AAIF,IAAM,gBAAgB,0BAA0B,OAAO;CACrD,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ;CACnB,eAAe;CACf,oBAAoB,iBAAiB,UAAU;CAC/C,mBAAmB,iBAAiB,UAAU;CAC/C,CAAC;AAOF,SAAS,aAAa,OAA4B;CAChD,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;AACxC,QACE,6BAA6B,MAAM,GAC3B,UAAU,OAAO;;AAK7B,eAAsB,WACpB,KACA,OACA,MACA,gBACA,EAAC,QAAQ,WACT,oBACA;CACA,MAAM,SAAS,eAAe,MAAM;CACpC,MAAM,SAA0B;EAAC;EAAQ;EAAQ;AACjD,OAAM,GAAG;kBACO,IAAI,OAAO,CAAC;;gBAEd,KAAK,IAAI,eAAe,IAAI,OAAO,IAAI,mBAAmB;;AAG1E,eAAsB,oBACpB,IACA,KACA,OACA,gBACA,SACyB;CACzB,MAAM,SAAS,IAAI,eAAe,MAAM,CAAC;CACzC,MAAM,SAAS,MAAM,GAAG;oBACN,OAAO,iBAAiB,OAAO;wBAC3B,eAAe;;AAErC,KAAI,OAAO,WAAW,GAAG;EAEvB,MAAM,cAAc,MAAM,GAAG;;eAElB,OAAO;AAClB,KAAG,OACD,WAAW,eAAe,MACvB,UAAU,aAAa,UAAU,QAAQ,CAAC,MAAM,MACjD,iBAAiB,UAAU,YAAY,GAC1C;AACD,SAAO;;AAET,QAAO,MAAQ,OAAO,IAAI,eAAe,cAAc;;AAGzD,eAAsB,uBACpB,KACA,OAC8B;CAC9B,MAAM,SAAS,MAAM,GAAG;;aAEb,IAAI,eAAe,MAAM,CAAC,CAAC;;AAEtC,QACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,SAC3D;AACD,QAAO,MAAQ,OAAO,IAAI,2BAA2B,cAAc;;;;;;AAOrE,eAAsB,0BACpB,IACA,KACA,WACA;CACA,MAAM,EAAC,iBAAgB;AAEvB,MAAK,MAAM,OAAO,cAAc;AAC9B,0BAAwB,IAAI;AAC5B,MAAI,IAAI,WAAW,IAAI,CACrB,OAAM,IAAI,MACR,oHACkD,IAAI,IACvD;;CAGL,MAAM,kBAA4B,EAAE;AAGpC,KAAI,aAAa,QAAQ;EACvB,MAAM,UAAU,MAAM,GAAwB;0DACQ,IACpD,aACD,GAAG,QAAQ;AAEZ,MAAI,QAAQ,WAAW,aAAa,OAClC,OAAM,IAAI,MACR,gDAAgD,aAAa,aAAa,QAAQ,MAAM,CAAC,GAC1F;AAEH,kBAAgB,KAAK,GAAG,aAAa;QAChC;EACL,MAAM,qBAAqB,uBACzB,UAAU,OACV,UAAU,SACX;AACD,QAAM,GAAG;mCACsB,IAAI,mBAAmB;AACtD,QAAM,GAAG;2BACc,IAAI,mBAAmB,CAAC;;;AAG/C,kBAAgB,KAAK,mBAAmB;;CAG1C,MAAM,sBAAsB,wBAC1B,UAAU,OACV,UAAU,SACX;AACD,iBAAgB,KAAK,oBAAoB;CAEzC,MAAM,QAAQ;EAAC,GAAG;EAAW,cAAc;EAAgB;AAG3D,OAAM,IAAI,OAAO,YAAY,MAAM,GAAG,WAAW,OAAO,oBAAoB,CAAC;AAG7E,OAAM,6CADO,MAAM,mBAAmB,KAAK,gBAAgB,CACH,EAAE,MAAM,IAAI,IAAI;AAExE,OAAM,cAAc,IAAI,KAAK,MAAM;;AAGrC,eAAsB,cACpB,IACA,IACA,OACA;AACA,KAAI;AACF,QAAM,GAAG,WAAU,QAAO,IAAI,OAAO,aAAa,MAAM,CAAC,CAAC;UACnD,GAAG;AACV,MACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,2BAGb,OAAM;AAGR,KAAG,OACD,oEACM,EAAE,QAAQ,EAAE,QAAQ,6IAG3B;;;AAIL,SAAgB,qBACd,IACA,WACA;AAEA,WAAU,aAAa,SAAQ,QAAO;AACpC,MACE,CAAC,IAAI,aACL,CAAC,IAAI,aACL,CAAC,IAAI,aACL,CAAC,IAAI,YAGL,OAAM,IAAI,MACR,eAAe,IAAI,QAAQ,oDAC5B;GAEH;AAEF,WAAU,OAAO,SAAQ,UAAS,SAAS,IAAI,MAAM,CAAC;;AAOxD,SAAgB,6CACd,MAC+B;CAC/B,MAAM,oBAIA,EAAE;AACR,MAAK,MAAM,SAAS,KAAK,OACvB,KAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAA,KAA6B;EAQlE,MAAM,EAAC,QAAQ,MAAM,cAAa;AAClC,OAAK,MAAM,EAAC,SAAS,MAAM,eAAc,KAAK,QAAQ,QACpD,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI,YACP,EAAE;AACD,OAAI,OAAO,KAAK,QAAQ,CAAC,MAAK,QAAO,CAAC,MAAM,QAAQ,KAAK,QAAQ,CAC/D;AAEF,qBAAkB,KAAK;IAAC;IAAQ;IAAW;IAAU,CAAC;AACtD;;;AAKN,KAAI,kBAAkB,WAAW,EAC/B;AAEF,QAAO,EACL,OAAO,OAAO,IAAgB,QAAoB;AAChD,OAAK,MAAM,EAAC,QAAQ,WAAW,eAAc,mBAAmB;AAC9D,MAAG,OACD,YAAY,UAAU,iCAAiC,UAAU,GAClE;AACD,SAAM,GAAG;sBACK,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,CAAC;yCACX,IAAI,UAAU;;IAGpD"}

package/out/zero-cache/src/services/change-streamer/broadcast.js CHANGED Viewed

@@ -152,7 +152,7 @@ var Broadcast = class {
 				processed: d.changes,
 				elapsed: d.elapsed
 			})),
-			pending: [...this.#pending].map((sub) => ({
+			pending: Array.from(this.#pending, (sub) => ({
 				id: sub.id,
 				...sub.getStats()
 			}))

package/out/zero-cache/src/services/change-streamer/broadcast.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"broadcast.js","names":["#pending","#completed","#done","#watermark","#majority","#start","#markCompleted","#setDone","#latestCompleted","#isDone","#logWithState"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\n/*\n Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers.\n \n Creating a `Broadcast` automatically initiates the send.\n \n By default, {@link Broadcast.done} resolves when all subscribers\n * have acked the change. However, {@link Broadcast.checkProgress()}\n * can be called to resolve the broadcast earlier based on the flow\n * control policy.\n /\nexport class Broadcast {\n /\n Sends the change to the subscribers without the tracking machinery.\n * This is suitable for fire-and-forget (i.e. pipelined) sends.\n /\n static withoutTracking(\n subscribers: Iterable<Subscriber>,\n change: WatermarkedChange,\n ) {\n for (const sub of subscribers) {\n void sub.send(change);\n }\n }\n\n readonly #pending: Set<Subscriber>;\n readonly #completed: Completed[];\n readonly #done = resolver();\n #isDone = false;\n\n readonly #watermark: string;\n readonly #majority: number;\n\n readonly #start = performance.now();\n #latestCompleted = Number.MAX_VALUE;\n\n /\n Broadcasts the `change` to the `subscribers` and tracks their\n * completion.\n /\n constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange) {\n this.#pending = new Set(subscribers);\n this.#completed = [];\n this.#watermark = change[0];\n this.#majority = Math.floor(this.#pending.size / 2) + 1;\n\n for (const sub of this.#pending) {\n const changes = sub.numPending + 1; // add one for this `change`\n void sub\n .send(change)\n .catch(() => {})\n .finally(() => this.#markCompleted(sub, changes));\n }\n\n // set done if there are no subscribers (mainly for tests)\n if (this.#pending.size === 0) {\n this.#setDone();\n }\n }\n\n #markCompleted(sub: Subscriber, changes: number) {\n const elapsed = (this.#latestCompleted = performance.now()) - this.#start;\n this.#completed.push({sub, changes, elapsed});\n this.#pending.delete(sub);\n if (this.#pending.size === 0) {\n this.#setDone();\n }\n }\n\n #setDone() {\n this.#isDone = true;\n this.#done.resolve();\n }\n\n get isDone(): boolean {\n return this.#isDone;\n }\n\n get done(): Promise<void> {\n return this.#done.promise;\n }\n\n /\n Checks for pathological situations in which flow should be reenabled\n * before all subscribers have acked.\n \n ### Background\n \n The purpose of flow control is to pull upstream replication changes\n * no faster than the rate as they are processed by downstream subscribers\n * in the steady state. In the change-streamer, this is done by occasionally\n * waiting for ACKs from subscribers before continuing; without doing so,\n * I/O buffers fill up and cause the system to spend most of its time in GC.\n \n However, the naive algorithm of always waiting for all subscribers (e.g.\n * `Promise.all()`) can behave poorly in scenarios where subscribers\n * are imbalanced:\n * * New subscribers may have a backlog of changes to catch up with.\n * Having all subscribers wait for the new subscriber to catch up results\n * in delaying the entire application.\n * * Broken TCP connections similarly require all subscribers to wait until\n * connection liveness checks kick in and disconnect the subscriber.\n \n A simplistic approach is to add a limit to the amount of time waiting for\n * subscribers, i.e. an ack timeout. However, deciding what this timeout\n * should be is non-trivial because of the heterogeneous nature of changes;\n * while most changes operate on single rows and are relatively predictable\n * in terms of running time, some changes are table-wide operations and can\n * legitimately take an arbitrary amount of time. In such scenarios, a\n * timeout that is too short can stop progress on replication altogether.\n \n ### Consensus-based Timeout Algorithm\n \n To address these shortcomings, a \"consensus-based timeout\" algorithm is\n * used:\n * * Wait for more than half of the subscribers to finish. (In\n * case of a single node, or the case of one replication-manager\n * and one view-syncer, this reduces to waiting for all subscribers.)\n * * Once more than half of the subscribers have finished, proceed after\n * a fixed timeout elapses (e.g. 1 second), even if not all subscribers\n * have finished.\n \n In other words, the subscribers themselves are used to determine the\n * timeout of each batch of changes; the majority determines this when\n * they complete, upon which a timeout is logically started.\n \n In the common case, the remaining subscribers finish soon afterward and\n * the timeout never elapses. However, in pathological cases where a minority\n * of subscribers have a disproportionate amount of load, some will still\n * be processing (or otherwise unresponsive). These subscribers are given\n * a bounded amount of time to catch up at each flushed batch, up to the\n * timeout interval. This guarantees eventual catchup because the\n * subscribers with a backlog of changes necessarily have a higher\n * processing rate than the subscribers that finished (and are made to wait).\n \n ### Not implemented: Broken connection detection\n \n If a subscriber has not made progress for a certain interval, the\n * algorithm could theoretically drop it preemptively, supplementing the\n * existing websocket-level liveness checks.\n \n However, a more reliable approach would be to change the replicator\n * to use non-blocking writes, and subsequently increase the frequency of\n * connection-level liveness checks. The current synchronous replica writes\n * can delay both ping responsiveness and change progress arbitrarily (e.g.\n * a large index creation); an independently liveness check that is not\n * delayed by synchronous writes on the subscriber would be a more failsafe\n * solution.\n \n @returns `true` if the broadcast was already done or was marked done.\n /\n checkProgress(\n lc: LogContext,\n flowControlConsensusPaddingMs: number,\n now: number,\n ) {\n if (this.#pending.size === 0) {\n return true;\n }\n const elapsed = now - this.#start;\n if (this.#completed.length < this.#majority) {\n if (elapsed >= 1000) {\n this.#logWithState(\n lc,\n `waiting for at least ${this.#majority} subscribers to finish`,\n elapsed,\n );\n }\n return false;\n }\n // Note: In the implementation, #latestCompleted is always updated,\n // even after the majority is reached. This is fine and does not affect\n // the important properties of the algorithm.\n if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {\n this.#logWithState(\n lc,\n `continuing with ${this.#pending.size} subscriber(s) still pending`,\n elapsed,\n );\n this.#setDone();\n return true;\n }\n return false;\n }\n\n #logWithState(lc: LogContext, msg: string, elapsed: number) {\n lc.withContext('watermark', this.#watermark).info?.(\n `${msg} (${elapsed.toFixed(3)} ms)`,\n {\n completed: this.#completed.map(d => ({\n id: d.sub.id,\n processed: d.changes,\n elapsed: d.elapsed,\n })),\n pending: ~~[...~~this.#pending~~].map(~~sub => ({\n id: sub.id,\n ...sub.getStats(),\n })),\n },\n );\n }\n}\n\n/* Tracks the completed result of a single subscriber. /\ntype Completed = {\n sub: Subscriber;\n /* The number of changes processed. /\n changes: number;\n /* The elapsed milliseconds. */\n elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;AAgBA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;AACA,OAAK,MAAM,OAAO,YACX,KAAI,KAAK,OAAO;;CAIzB;CACA;CACA,QAAiB,UAAU;CAC3B,UAAU;CAEV;CACA;CAEA,SAAkB,YAAY,KAAK;CACnC,mBAAmB,OAAO;;;;;CAM1B,YAAY,aAAmC,QAA2B;AACxE,QAAA,UAAgB,IAAI,IAAI,YAAY;AACpC,QAAA,YAAkB,EAAE;AACpB,QAAA,YAAkB,OAAO;AACzB,QAAA,WAAiB,KAAK,MAAM,MAAA,QAAc,OAAO,EAAE,GAAG;AAEtD,OAAK,MAAM,OAAO,MAAA,SAAe;GAC/B,MAAM,UAAU,IAAI,aAAa;AAC5B,OACF,KAAK,OAAO,CACZ,YAAY,GAAG,CACf,cAAc,MAAA,cAAoB,KAAK,QAAQ,CAAC;;AAIrD,MAAI,MAAA,QAAc,SAAS,EACzB,OAAA,SAAe;;CAInB,eAAe,KAAiB,SAAiB;EAC/C,MAAM,WAAW,MAAA,kBAAwB,YAAY,KAAK,IAAI,MAAA;AAC9D,QAAA,UAAgB,KAAK;GAAC;GAAK;GAAS;GAAQ,CAAC;AAC7C,QAAA,QAAc,OAAO,IAAI;AACzB,MAAI,MAAA,QAAc,SAAS,EACzB,OAAA,SAAe;;CAInB,WAAW;AACT,QAAA,SAAe;AACf,QAAA,KAAW,SAAS;;CAGtB,IAAI,SAAkB;AACpB,SAAO,MAAA;;CAGT,IAAI,OAAsB;AACxB,SAAO,MAAA,KAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwEpB,cACE,IACA,+BACA,KACA;AACA,MAAI,MAAA,QAAc,SAAS,EACzB,QAAO;EAET,MAAM,UAAU,MAAM,MAAA;AACtB,MAAI,MAAA,UAAgB,SAAS,MAAA,UAAgB;AAC3C,OAAI,WAAW,IACb,OAAA,aACE,IACA,wBAAwB,MAAA,SAAe,yBACvC,QACD;AAEH,UAAO;;AAKT,MAAI,MAAM,MAAA,mBAAyB,+BAA+B;AAChE,SAAA,aACE,IACA,mBAAmB,MAAA,QAAc,KAAK,+BACtC,QACD;AACD,SAAA,SAAe;AACf,UAAO;;AAET,SAAO;;CAGT,cAAc,IAAgB,KAAa,SAAiB;AAC1D,KAAG,YAAY,aAAa,MAAA,UAAgB,CAAC,OAC3C,GAAG,IAAI,IAAI,QAAQ,QAAQ,EAAE,CAAC,OAC9B;GACE,WAAW,MAAA,UAAgB,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;IACZ,EAAE;GACH,SAAS,~~CAAC~~,~~GAAG~~,MAAA,~~QAAc~~,~~CAAC,KAAI,~~SAAQ;~~IACtC~~,IAAI,IAAI;IACR,GAAG,IAAI,UAAU;IAClB,EAAE;GACJ,CACF"}
1	+ {"version":3,"file":"broadcast.js","names":["#pending","#completed","#done","#watermark","#majority","#start","#markCompleted","#setDone","#latestCompleted","#isDone","#logWithState"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\n/*\n Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers.\n \n Creating a `Broadcast` automatically initiates the send.\n \n By default, {@link Broadcast.done} resolves when all subscribers\n * have acked the change. However, {@link Broadcast.checkProgress()}\n * can be called to resolve the broadcast earlier based on the flow\n * control policy.\n /\nexport class Broadcast {\n /\n Sends the change to the subscribers without the tracking machinery.\n * This is suitable for fire-and-forget (i.e. pipelined) sends.\n /\n static withoutTracking(\n subscribers: Iterable<Subscriber>,\n change: WatermarkedChange,\n ) {\n for (const sub of subscribers) {\n void sub.send(change);\n }\n }\n\n readonly #pending: Set<Subscriber>;\n readonly #completed: Completed[];\n readonly #done = resolver();\n #isDone = false;\n\n readonly #watermark: string;\n readonly #majority: number;\n\n readonly #start = performance.now();\n #latestCompleted = Number.MAX_VALUE;\n\n /\n Broadcasts the `change` to the `subscribers` and tracks their\n * completion.\n /\n constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange) {\n this.#pending = new Set(subscribers);\n this.#completed = [];\n this.#watermark = change[0];\n this.#majority = Math.floor(this.#pending.size / 2) + 1;\n\n for (const sub of this.#pending) {\n const changes = sub.numPending + 1; // add one for this `change`\n void sub\n .send(change)\n .catch(() => {})\n .finally(() => this.#markCompleted(sub, changes));\n }\n\n // set done if there are no subscribers (mainly for tests)\n if (this.#pending.size === 0) {\n this.#setDone();\n }\n }\n\n #markCompleted(sub: Subscriber, changes: number) {\n const elapsed = (this.#latestCompleted = performance.now()) - this.#start;\n this.#completed.push({sub, changes, elapsed});\n this.#pending.delete(sub);\n if (this.#pending.size === 0) {\n this.#setDone();\n }\n }\n\n #setDone() {\n this.#isDone = true;\n this.#done.resolve();\n }\n\n get isDone(): boolean {\n return this.#isDone;\n }\n\n get done(): Promise<void> {\n return this.#done.promise;\n }\n\n /\n Checks for pathological situations in which flow should be reenabled\n * before all subscribers have acked.\n \n ### Background\n \n The purpose of flow control is to pull upstream replication changes\n * no faster than the rate as they are processed by downstream subscribers\n * in the steady state. In the change-streamer, this is done by occasionally\n * waiting for ACKs from subscribers before continuing; without doing so,\n * I/O buffers fill up and cause the system to spend most of its time in GC.\n \n However, the naive algorithm of always waiting for all subscribers (e.g.\n * `Promise.all()`) can behave poorly in scenarios where subscribers\n * are imbalanced:\n * * New subscribers may have a backlog of changes to catch up with.\n * Having all subscribers wait for the new subscriber to catch up results\n * in delaying the entire application.\n * * Broken TCP connections similarly require all subscribers to wait until\n * connection liveness checks kick in and disconnect the subscriber.\n \n A simplistic approach is to add a limit to the amount of time waiting for\n * subscribers, i.e. an ack timeout. However, deciding what this timeout\n * should be is non-trivial because of the heterogeneous nature of changes;\n * while most changes operate on single rows and are relatively predictable\n * in terms of running time, some changes are table-wide operations and can\n * legitimately take an arbitrary amount of time. In such scenarios, a\n * timeout that is too short can stop progress on replication altogether.\n \n ### Consensus-based Timeout Algorithm\n \n To address these shortcomings, a \"consensus-based timeout\" algorithm is\n * used:\n * * Wait for more than half of the subscribers to finish. (In\n * case of a single node, or the case of one replication-manager\n * and one view-syncer, this reduces to waiting for all subscribers.)\n * * Once more than half of the subscribers have finished, proceed after\n * a fixed timeout elapses (e.g. 1 second), even if not all subscribers\n * have finished.\n \n In other words, the subscribers themselves are used to determine the\n * timeout of each batch of changes; the majority determines this when\n * they complete, upon which a timeout is logically started.\n \n In the common case, the remaining subscribers finish soon afterward and\n * the timeout never elapses. However, in pathological cases where a minority\n * of subscribers have a disproportionate amount of load, some will still\n * be processing (or otherwise unresponsive). These subscribers are given\n * a bounded amount of time to catch up at each flushed batch, up to the\n * timeout interval. This guarantees eventual catchup because the\n * subscribers with a backlog of changes necessarily have a higher\n * processing rate than the subscribers that finished (and are made to wait).\n \n ### Not implemented: Broken connection detection\n \n If a subscriber has not made progress for a certain interval, the\n * algorithm could theoretically drop it preemptively, supplementing the\n * existing websocket-level liveness checks.\n \n However, a more reliable approach would be to change the replicator\n * to use non-blocking writes, and subsequently increase the frequency of\n * connection-level liveness checks. The current synchronous replica writes\n * can delay both ping responsiveness and change progress arbitrarily (e.g.\n * a large index creation); an independently liveness check that is not\n * delayed by synchronous writes on the subscriber would be a more failsafe\n * solution.\n \n @returns `true` if the broadcast was already done or was marked done.\n /\n checkProgress(\n lc: LogContext,\n flowControlConsensusPaddingMs: number,\n now: number,\n ) {\n if (this.#pending.size === 0) {\n return true;\n }\n const elapsed = now - this.#start;\n if (this.#completed.length < this.#majority) {\n if (elapsed >= 1000) {\n this.#logWithState(\n lc,\n `waiting for at least ${this.#majority} subscribers to finish`,\n elapsed,\n );\n }\n return false;\n }\n // Note: In the implementation, #latestCompleted is always updated,\n // even after the majority is reached. This is fine and does not affect\n // the important properties of the algorithm.\n if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {\n this.#logWithState(\n lc,\n `continuing with ${this.#pending.size} subscriber(s) still pending`,\n elapsed,\n );\n this.#setDone();\n return true;\n }\n return false;\n }\n\n #logWithState(lc: LogContext, msg: string, elapsed: number) {\n lc.withContext('watermark', this.#watermark).info?.(\n `${msg} (${elapsed.toFixed(3)} ms)`,\n {\n completed: this.#completed.map(d => ({\n id: d.sub.id,\n processed: d.changes,\n elapsed: d.elapsed,\n })),\n pending: Array.from(this.#pending, sub => ({\n id: sub.id,\n ...sub.getStats(),\n })),\n },\n );\n }\n}\n\n/* Tracks the completed result of a single subscriber. /\ntype Completed = {\n sub: Subscriber;\n /* The number of changes processed. /\n changes: number;\n /* The elapsed milliseconds. */\n elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;AAgBA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;AACA,OAAK,MAAM,OAAO,YACX,KAAI,KAAK,OAAO;;CAIzB;CACA;CACA,QAAiB,UAAU;CAC3B,UAAU;CAEV;CACA;CAEA,SAAkB,YAAY,KAAK;CACnC,mBAAmB,OAAO;;;;;CAM1B,YAAY,aAAmC,QAA2B;AACxE,QAAA,UAAgB,IAAI,IAAI,YAAY;AACpC,QAAA,YAAkB,EAAE;AACpB,QAAA,YAAkB,OAAO;AACzB,QAAA,WAAiB,KAAK,MAAM,MAAA,QAAc,OAAO,EAAE,GAAG;AAEtD,OAAK,MAAM,OAAO,MAAA,SAAe;GAC/B,MAAM,UAAU,IAAI,aAAa;AAC5B,OACF,KAAK,OAAO,CACZ,YAAY,GAAG,CACf,cAAc,MAAA,cAAoB,KAAK,QAAQ,CAAC;;AAIrD,MAAI,MAAA,QAAc,SAAS,EACzB,OAAA,SAAe;;CAInB,eAAe,KAAiB,SAAiB;EAC/C,MAAM,WAAW,MAAA,kBAAwB,YAAY,KAAK,IAAI,MAAA;AAC9D,QAAA,UAAgB,KAAK;GAAC;GAAK;GAAS;GAAQ,CAAC;AAC7C,QAAA,QAAc,OAAO,IAAI;AACzB,MAAI,MAAA,QAAc,SAAS,EACzB,OAAA,SAAe;;CAInB,WAAW;AACT,QAAA,SAAe;AACf,QAAA,KAAW,SAAS;;CAGtB,IAAI,SAAkB;AACpB,SAAO,MAAA;;CAGT,IAAI,OAAsB;AACxB,SAAO,MAAA,KAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwEpB,cACE,IACA,+BACA,KACA;AACA,MAAI,MAAA,QAAc,SAAS,EACzB,QAAO;EAET,MAAM,UAAU,MAAM,MAAA;AACtB,MAAI,MAAA,UAAgB,SAAS,MAAA,UAAgB;AAC3C,OAAI,WAAW,IACb,OAAA,aACE,IACA,wBAAwB,MAAA,SAAe,yBACvC,QACD;AAEH,UAAO;;AAKT,MAAI,MAAM,MAAA,mBAAyB,+BAA+B;AAChE,SAAA,aACE,IACA,mBAAmB,MAAA,QAAc,KAAK,+BACtC,QACD;AACD,SAAA,SAAe;AACf,UAAO;;AAET,SAAO;;CAGT,cAAc,IAAgB,KAAa,SAAiB;AAC1D,KAAG,YAAY,aAAa,MAAA,UAAgB,CAAC,OAC3C,GAAG,IAAI,IAAI,QAAQ,QAAQ,EAAE,CAAC,OAC9B;GACE,WAAW,MAAA,UAAgB,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;IACZ,EAAE;GACH,SAAS,MAAM,KAAK,MAAA,UAAe,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,UAAU;IAClB,EAAE;GACJ,CACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js CHANGED Viewed

@@ -179,6 +179,7 @@ var ChangeStreamerImpl = class {
 	#initialWatermarks = /* @__PURE__ */ new Set();
 	#serving = resolver();
 	#txCounter = getOrCreateCounter("replication", "transactions", "Count of replicated transactions");
+	#changeCounter = getOrCreateCounter("replication", "changes", "Count of replicated changes (DML or DDL statements)");
 	#latestStatus;
 	#stream;
 	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, source, replicationStatusPublisher, autoReset, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds, setTimeoutFn = setTimeout) {
@@ -238,6 +239,7 @@ var ChangeStreamerImpl = class {
 							this.#txCounter.add(1);
 							break;
 						default:
+							if (type === "data") this.#changeCounter.add(1);
 							if (watermark === null) throw new UnrecoverableError(`${type} change (${msg.tag}) received before 'begin' message`);
 							break;
 					}
@@ -353,6 +355,7 @@ var ChangeStreamerImpl = class {
 		this.#state.stop(this.#lc, err);
 		this.#stream?.changes.cancel();
 		await this.#storer.stop();
+		await this.#source.stop();
 	}
 };
 var CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#stream","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n min,\n type AtLeastOne,\n type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n ChangeSource,\n ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n type ChangeStreamControl,\n type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n publishReplicationError,\n replicationStatusError,\n type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n DEFAULT_MAX_RETRY_DELAY_MS,\n RunningState,\n UnrecoverableError,\n} from '../running-state.ts';\nimport {\n type ChangeStreamerService,\n type Downstream,\n type Status,\n type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n AutoResetSignal,\n ensureReplicationConfig,\n markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/*\n Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n /\nexport async function initializeStreamer(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n changeSource: ChangeSource,\n replicationStatusPublisher: ReplicationStatusPublisher,\n subscriptionState: SubscriptionState,\n autoReset: boolean,\n backPressureLimitHeapProportion: number,\n flowControlConsensusPaddingSeconds: number,\n setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n // Make sure the ChangeLog DB is set up.\n await initChangeStreamerSchema(lc, changeDB, shard);\n await ensureReplicationConfig(\n lc,\n changeDB,\n subscriptionState,\n shard,\n autoReset,\n setTimeoutFn,\n );\n\n const {replicaVersion} = subscriptionState;\n return new ChangeStreamerImpl(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n changeSource,\n replicationStatusPublisher,\n autoReset,\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n setTimeoutFn,\n );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\n/\n Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n \n However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n /\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/\n Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n \n ### Subscriber Catchup\n \n Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n \n \n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * \| Historic changes in storage \| Pending (streamed) tx \| Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * Replication stream\n * > > > > > > > > >\n * ^ ---> required catchup ---> ^\n * Subscriber watermark Subscription begins\n * ```\n \n Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n \n Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n \n * If the streamer is in the middle of a pending Transaction, the\n * Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n * state, new changes are not forwarded to the Subscriber, and catchup\n * is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n * by the Storer, it begins catchup on the Subscriber (with a READONLY\n * snapshot so that it does not block subsequent storage operations).\n * This catchup is thus guaranteed to load the change log entries of\n * that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n * Subscriber from the \"queued\" to the \"active\" set of clients such that\n * the Subscriber begins receiving new changes, starting from the next\n * Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n * is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n * and the Subscriber henceforth forwards changes as they are received.\n \n In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n \n ### Watermarks and ordering\n \n The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n \n https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n \n > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n \n In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n \n A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n \n ```\n * LSN: 1 2 3 4 5 6 7 8 9 10\n * tx1: begin data data data commit\n * tx2: begin data data data commit\n * ```\n \n will arrive as:\n \n ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n \n Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n \n Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n \n * A `commit` message is assigned to a watermark corresponding to its LSN.\n * These are guaranteed to be in commit order by definition.\n \n * `begin` and `data` messages are assigned to the watermark of the\n * preceding `commit` (the previous transaction, or the replication\n * slot's starting LSN) plus 1. This guarantees that they will be sorted\n * after the previously commit transaction even if their LSNs came before it.\n * This is referred to as the `preCommitWatermark`.\n \n * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n * the position of the message within its transaction, with the `begin` message\n * starting at `0`. This guarantees that `begin` and `data` messages will be\n * fetched in the original ChangeSource order during catchup.\n \n `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n \n ### Cleanup\n \n As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n \n More concretely:\n \n * The `initial`, backup-derived watermark is the earliest to which cleanup\n * should ever happen.\n \n * However, it is possible for the replica backup to be ahead of a connected\n * subscriber; and if a network error causes that subscriber to retry from its\n * last watermark, the change streamer must support it.\n \n Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n /\nclass ChangeStreamerImpl implements ChangeStreamerService {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #replicaVersion: string;\n readonly #source: ChangeSource;\n readonly #storer: Storer;\n readonly #forwarder: Forwarder;\n readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n readonly #autoReset: boolean;\n readonly #state: RunningState;\n readonly #initialWatermarks = new Set<string>();\n\n // Starting the (Postgres) ChangeStream results in killing the previous\n // Postgres subscriber, potentially creating a gap in which the old\n // change-streamer has shut down and the new change-streamer has not yet\n // been recognized as \"healthy\" (and thus does not get any requests).\n //\n // To minimize this gap, delay starting the ChangeStream until the first\n // request from a `serving` replicator, indicating that higher level\n // load-balancing / routing logic has begun routing requests to this task.\n readonly #serving = resolver();\n\n readonly #txCounter = getOrCreateCounter(\n 'replication',\n 'transactions',\n 'Count of replicated transactions',\n );\n\n #latestStatus: Status;\n #stream: ChangeStream \| undefined;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n replicaVersion: string,\n source: ChangeSource,\n replicationStatusPublisher: ReplicationStatusPublisher,\n autoReset: boolean,\n backPressureLimitHeapProportion: number,\n flowControlConsensusPaddingSeconds: number,\n setTimeoutFn = setTimeout,\n ) {\n this.id = `change-streamer`;\n this.#lc = lc.withContext('component', 'change-streamer');\n this.#shard = shard;\n this.#changeDB = changeDB;\n this.#replicaVersion = replicaVersion;\n this.#source = source;\n this.#storer = new Storer(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n err => this.stop(err),\n backPressureLimitHeapProportion,\n );\n this.#forwarder = new Forwarder(lc, {\n flowControlConsensusPaddingSeconds,\n });\n this.#replicationStatusPublisher = replicationStatusPublisher;\n this.#autoReset = autoReset;\n this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n this.#latestStatus = {tag: 'status'};\n }\n\n async run() {\n this.#lc.info?.('starting change stream');\n\n this.#forwarder.startProgressMonitor();\n\n const lagReport = await this.#source.startLagReporter();\n if (lagReport) {\n this.#latestStatus.lagReport = lagReport;\n }\n\n // Once this change-streamer acquires \"ownership\" of the change DB,\n // it is safe to start the storer.\n await this.#storer.assumeOwnership();\n\n // The threshold in (estimated number of) bytes to send() on subscriber\n // websockets before `await`-ing the I/O buffers to be ready for more.\n const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n while (this.#state.shouldRun()) {\n let err: unknown;\n let watermark: string \| null = null;\n let unflushedBytes = 0;\n try {\n const {lastWatermark, backfillRequests} =\n await this.#storer.getStartStreamInitializationParameters();\n const stream = await this.#source.startStream(\n lastWatermark,\n backfillRequests,\n );\n this.#storer.run().catch(e => stream.changes.cancel(e));\n\n this.#stream = stream;\n if (\n this.#state.resetBackoff() >\n REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n ) {\n // After recovering from a backoff for which a replication status\n // error was published, publish an OK status\n this.#replicationStatusPublisher.publish(\n this.#lc,\n 'Replicating',\n `Replicating from ${lastWatermark}`,\n );\n }\n watermark = null;\n\n for await (const change of stream.changes) {\n const [type, msg] = change;\n switch (type) {\n case 'status':\n if (msg.ack) {\n this.#storer.status(change); // storer acks once it gets through its queue\n }\n if (msg.lagReport) {\n // Lag reports are not stored in the cdc change log, but rather\n // only forwarded on \"live\" connections. When a new subscriber\n // is catching up, it is initialized with the #latestStatus\n // from which it can measure lag while catching up.\n this.#latestStatus.lagReport = msg.lagReport;\n this.#forwarder.sendStatus(this.#latestStatus);\n }\n continue;\n case 'control':\n await this.#handleControlMessage(msg);\n continue; // control messages are not stored/forwarded\n case 'begin':\n watermark = change[2].commitWatermark;\n break;\n case 'commit':\n if (watermark !== change[2].watermark) {\n throw new UnrecoverableError(\n `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n );\n }\n this.#txCounter.add(1);\n break;\n default:\n if (watermark === null) {\n throw new UnrecoverableError(\n `${type} change (${msg.tag}) received before 'begin' message`,\n );\n }\n break;\n }\n\n const entry: WatermarkedChange = [watermark, change];\n unflushedBytes += this.#storer.store(entry);\n if (unflushedBytes < flushBytesThreshold) {\n // pipeline changes until flushBytesThreshold\n this.#forwarder.forward(entry);\n } else {\n // Wait for messages to clear socket buffers to ensure that they\n // make their way to subscribers. Without this `await`, the\n // messages end up being buffered in this process, which:\n // (1) results in memory pressure and increased GC activity\n // (2) prevents subscribers from processing the messages as they\n // arrive, instead getting them in a large batch after being\n // idle while they were queued (causing further delays).\n await this.#forwarder.forwardWithFlowControl(entry);\n unflushedBytes = 0;\n }\n\n if (type === 'commit' \|\| type === 'rollback') {\n watermark = null;\n }\n\n // Allow the storer to exert back pressure.\n const readyForMore = this.#storer.readyForMore();\n if (readyForMore) {\n await readyForMore;\n }\n }\n } catch (e) {\n err = e;\n } finally {\n this.#stream?.changes.cancel();\n this.#stream = undefined;\n }\n\n // When the change stream is interrupted, abort any pending transaction.\n if (watermark) {\n this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n this.#storer.abort();\n this.#forwarder.forward([watermark, ['rollback', {tag: 'rollback'}]]);\n }\n\n // Backoff and drain any pending entries in the storer before reconnecting.\n await Promise.all([\n this.#storer.stop(),\n this.#state.backoff(this.#lc, err),\n this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n ? publishCriticalEvent(\n this.#lc,\n replicationStatusError(this.#lc, 'Replicating', err),\n )\n : promiseVoid,\n ]);\n }\n\n this.#forwarder.stopProgressMonitor();\n this.#lc.info?.('ChangeStreamer stopped');\n }\n\n async #handleControlMessage(msg: ChangeStreamControl[1]) {\n this.#lc.info?.('received control message', msg);\n const {tag} = msg;\n\n switch (tag) {\n case 'reset-required':\n await markResetRequired(this.#changeDB, this.#shard);\n await publishReplicationError(\n this.#lc,\n 'Replicating',\n msg.message ?? 'Resync required',\n msg.errorDetails,\n );\n if (this.#autoReset) {\n this.#lc.warn?.('shutting down for auto-reset');\n await this.stop(new AutoResetSignal());\n }\n break;\n default:\n unreachable(tag);\n }\n }\n\n subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n if (mode === 'serving') {\n this.#serving.resolve();\n }\n const downstream = Subscription.create<Downstream>({\n cleanup: () => this.#forwarder.remove(subscriber),\n });\n const subscriber = new Subscriber(\n protocolVersion,\n id,\n watermark,\n downstream,\n () => this.#latestStatus,\n );\n if (replicaVersion !== this.#replicaVersion) {\n this.#lc.warn?.(\n `rejecting subscriber at replica version ${replicaVersion}`,\n );\n subscriber.close(\n ErrorType.WrongReplicaVersion,\n `current replica version is ${\n this.#replicaVersion\n } (requested ${replicaVersion})`,\n );\n } else {\n this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n this.#forwarder.add(subscriber);\n this.#storer.catchup(subscriber, mode);\n }\n return Promise.resolve(downstream);\n }\n\n scheduleCleanup(watermark: string) {\n const origSize = this.#initialWatermarks.size;\n this.#initialWatermarks.add(watermark);\n\n if (origSize === 0) {\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n\n async getChangeLogState(): Promise<{\n replicaVersion: string;\n minWatermark: string;\n }> {\n const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n if (!minWatermark) {\n this.#lc.warn?.(\n `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n );\n }\n return {\n replicaVersion: this.#replicaVersion,\n minWatermark: minWatermark ?? this.#replicaVersion,\n };\n }\n\n /\n Makes a best effort to purge the change log. In the event of a database\n * error, exceptions will be logged and swallowed, so this method is safe\n * to run in a timeout.\n /\n async #purgeOldChanges(): Promise<void> {\n const initial = [...this.#initialWatermarks];\n if (initial.length === 0) {\n this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n return;\n }\n const current = [...this.#forwarder.getAcks()];\n if (current.length === 0) {\n // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n // Bail to be safe.\n this.#lc.warn?.('No subscribers to confirm cleanup');\n return;\n }\n try {\n const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n if (earliestCurrent < earliestInitial) {\n this.#lc.info?.(\n `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n );\n } else {\n this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n const start = performance.now();\n const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n );\n this.#initialWatermarks.delete(earliestInitial);\n }\n } catch (e) {\n this.#lc.warn?.(`error purging change log`, e);\n } finally {\n if (this.#initialWatermarks.size) {\n // If there are unpurged watermarks to check, schedule the next purge.\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n }\n\n async stop(err?: unknown) {\n this.#state.stop(this.#lc, err);\n this.#stream?.changes.cancel();\n await this.#storer.stop();\n }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n// `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS 3;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAuDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,iCACA,oCACA,eAAe,YACiB;AAEhC,OAAM,yBAAyB,IAAI,UAAU,MAAM;AACnD,OAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,aACD;CAED,MAAM,EAAC,mBAAkB;AACzB,QAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,iCACA,oCACA,aACD;;AAGH,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwJpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,KAAa;CAU/C,WAAoB,UAAU;CAE9B,aAAsB,mBACpB,eACA,gBACA,mCACD;CAED;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,WACA,iCACA,oCACA,eAAe,YACf;AACA,OAAK,KAAK;AACV,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AACzD,QAAA,QAAc;AACd,QAAA,WAAiB;AACjB,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,SAAe,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,MAAA,QAAc,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;GAAG,CAAC,GACzE,QAAO,KAAK,KAAK,IAAI,EACrB,gCACD;AACD,QAAA,YAAkB,IAAI,UAAU,IAAI,EAClC,oCACD,CAAC;AACF,QAAA,6BAAmC;AACnC,QAAA,YAAkB;AAClB,QAAA,QAAc,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,aAAa;AAChE,QAAA,eAAqB,EAAC,KAAK,UAAS;;CAGtC,MAAM,MAAM;AACV,QAAA,GAAS,OAAO,yBAAyB;AAEzC,QAAA,UAAgB,sBAAsB;EAEtC,MAAM,YAAY,MAAM,MAAA,OAAa,kBAAkB;AACvD,MAAI,UACF,OAAA,aAAmB,YAAY;AAKjC,QAAM,MAAA,OAAa,iBAAiB;EAIpC,MAAM,sBAAsB,wBAAwB,MAAM;AAE1D,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;AACrB,OAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,MAAA,OAAa,wCAAwC;IAC7D,MAAM,SAAS,MAAM,MAAA,OAAa,YAChC,eACA,iBACD;AACD,UAAA,OAAa,KAAK,CAAC,OAAM,MAAK,OAAO,QAAQ,OAAO,EAAE,CAAC;AAEvD,UAAA,SAAe;AACf,QACE,MAAA,MAAY,cAAc,GAC1B,4CAIA,OAAA,2BAAiC,QAC/B,MAAA,IACA,eACA,oBAAoB,gBACrB;AAEH,gBAAY;AAEZ,eAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;AACpB,aAAQ,MAAR;MACE,KAAK;AACH,WAAI,IAAI,IACN,OAAA,OAAa,OAAO,OAAO;AAE7B,WAAI,IAAI,WAAW;AAKjB,cAAA,aAAmB,YAAY,IAAI;AACnC,cAAA,UAAgB,WAAW,MAAA,aAAmB;;AAEhD;MACF,KAAK;AACH,aAAM,MAAA,qBAA2B,IAAI;AACrC;MACF,KAAK;AACH,mBAAY,OAAO,GAAG;AACtB;MACF,KAAK;AACH,WAAI,cAAc,OAAO,GAAG,UAC1B,OAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,YAC7E;AAEH,aAAA,UAAgB,IAAI,EAAE;AACtB;MACF;AACE,WAAI,cAAc,KAChB,OAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,mCAC5B;AAEH;;KAGJ,MAAM,QAA2B,CAAC,WAAW,OAAO;AACpD,uBAAkB,MAAA,OAAa,MAAM,MAAM;AAC3C,SAAI,iBAAiB,oBAEnB,OAAA,UAAgB,QAAQ,MAAM;UACzB;AAQL,YAAM,MAAA,UAAgB,uBAAuB,MAAM;AACnD,uBAAiB;;AAGnB,SAAI,SAAS,YAAY,SAAS,WAChC,aAAY;KAId,MAAM,eAAe,MAAA,OAAa,cAAc;AAChD,SAAI,aACF,OAAM;;YAGH,GAAG;AACV,UAAM;aACE;AACR,UAAA,QAAc,QAAQ,QAAQ;AAC9B,UAAA,SAAe,KAAA;;AAIjB,OAAI,WAAW;AACb,UAAA,GAAS,OAAO,oCAAoC,YAAY;AAChE,UAAA,OAAa,OAAO;AACpB,UAAA,UAAgB,QAAQ,CAAC,WAAW,CAAC,YAAY,EAAC,KAAK,YAAW,CAAC,CAAC,CAAC;;AAIvE,SAAM,QAAQ,IAAI;IAChB,MAAA,OAAa,MAAM;IACnB,MAAA,MAAY,QAAQ,MAAA,IAAU,IAAI;IAClC,MAAA,MAAY,aAAa,8CACrB,qBACE,MAAA,IACA,uBAAuB,MAAA,IAAU,eAAe,IAAI,CACrD,GACD;IACL,CAAC;;AAGJ,QAAA,UAAgB,qBAAqB;AACrC,QAAA,GAAS,OAAO,yBAAyB;;CAG3C,OAAA,qBAA4B,KAA6B;AACvD,QAAA,GAAS,OAAO,4BAA4B,IAAI;EAChD,MAAM,EAAC,QAAO;AAEd,UAAQ,KAAR;GACE,KAAK;AACH,UAAM,kBAAkB,MAAA,UAAgB,MAAA,MAAY;AACpD,UAAM,wBACJ,MAAA,IACA,eACA,IAAI,WAAW,mBACf,IAAI,aACL;AACD,QAAI,MAAA,WAAiB;AACnB,WAAA,GAAS,OAAO,+BAA+B;AAC/C,WAAM,KAAK,KAAK,IAAI,iBAAiB,CAAC;;AAExC;GACF,QACE,aAAY,IAAI;;;CAItB,UAAU,KAAqD;EAC7D,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;AAC/D,MAAI,SAAS,UACX,OAAA,QAAc,SAAS;EAEzB,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe,MAAA,UAAgB,OAAO,WAAW,EAClD,CAAC;EACF,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,MAAA,aACP;AACD,MAAI,mBAAmB,MAAA,gBAAsB;AAC3C,SAAA,GAAS,OACP,2CAA2C,iBAC5C;AACD,cAAW,MACT,GACA,8BACE,MAAA,eACD,cAAc,eAAe,GAC/B;SACI;AACL,SAAA,GAAS,QAAQ,qBAAqB,WAAW,KAAK;AAEtD,SAAA,UAAgB,IAAI,WAAW;AAC/B,SAAA,OAAa,QAAQ,YAAY,KAAK;;AAExC,SAAO,QAAQ,QAAQ,WAAW;;CAGpC,gBAAgB,WAAmB;EACjC,MAAM,WAAW,MAAA,kBAAwB;AACzC,QAAA,kBAAwB,IAAI,UAAU;AAEtC,MAAI,aAAa,EACf,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;CAI3E,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,MAAA,OAAa,2BAA2B;AACnE,MAAI,CAAC,aACH,OAAA,GAAS,OACP,+EACD;AAEH,SAAO;GACL,gBAAgB,MAAA;GAChB,cAAc,gBAAgB,MAAA;GAC/B;;;;;;;CAQH,OAAA,kBAAwC;EACtC,MAAM,UAAU,CAAC,GAAG,MAAA,kBAAwB;AAC5C,MAAI,QAAQ,WAAW,GAAG;AACxB,SAAA,GAAS,OAAO,6CAA6C;AAC7D;;EAEF,MAAM,UAAU,CAAC,GAAG,MAAA,UAAgB,SAAS,CAAC;AAC9C,MAAI,QAAQ,WAAW,GAAG;AAGxB,SAAA,GAAS,OAAO,oCAAoC;AACpD;;AAEF,MAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,QAAoC;GACpE,MAAM,kBAAkB,IAAI,GAAI,QAAoC;AACpE,OAAI,kBAAkB,gBACpB,OAAA,GAAS,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,GAC/E;QACI;AACL,UAAA,GAAS,OAAO,0BAA0B,gBAAgB,MAAM;IAChE,MAAM,QAAQ,YAAY,KAAK;IAC/B,MAAM,UAAU,MAAM,MAAA,OAAa,mBAAmB,gBAAgB;IACtE,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,UAAA,GAAS,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,MACjE;AACD,UAAA,kBAAwB,OAAO,gBAAgB;;WAE1C,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;YACtC;AACR,OAAI,MAAA,kBAAwB,KAE1B,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;;CAK7E,MAAM,KAAK,KAAe;AACxB,QAAA,MAAY,KAAK,MAAA,IAAU,IAAI;AAC/B,QAAA,QAAc,QAAQ,QAAQ;AAC9B,QAAM,MAAA,OAAa,MAAM;;;AAmB7B,IAAM,mBAAmB,6BAA6B"}
1	+ {"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#changeCounter","#stream","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n min,\n type AtLeastOne,\n type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n ChangeSource,\n ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n type ChangeStreamControl,\n type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n publishReplicationError,\n replicationStatusError,\n type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n DEFAULT_MAX_RETRY_DELAY_MS,\n RunningState,\n UnrecoverableError,\n} from '../running-state.ts';\nimport {\n type ChangeStreamerService,\n type Downstream,\n type Status,\n type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n AutoResetSignal,\n ensureReplicationConfig,\n markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/*\n Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n /\nexport async function initializeStreamer(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n changeSource: ChangeSource,\n replicationStatusPublisher: ReplicationStatusPublisher,\n subscriptionState: SubscriptionState,\n autoReset: boolean,\n backPressureLimitHeapProportion: number,\n flowControlConsensusPaddingSeconds: number,\n setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n // Make sure the ChangeLog DB is set up.\n await initChangeStreamerSchema(lc, changeDB, shard);\n await ensureReplicationConfig(\n lc,\n changeDB,\n subscriptionState,\n shard,\n autoReset,\n setTimeoutFn,\n );\n\n const {replicaVersion} = subscriptionState;\n return new ChangeStreamerImpl(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n changeSource,\n replicationStatusPublisher,\n autoReset,\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n setTimeoutFn,\n );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\n/\n Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n \n However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n /\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/\n Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n \n ### Subscriber Catchup\n \n Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n \n \n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * \| Historic changes in storage \| Pending (streamed) tx \| Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * Replication stream\n * > > > > > > > > >\n * ^ ---> required catchup ---> ^\n * Subscriber watermark Subscription begins\n * ```\n \n Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n \n Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n \n * If the streamer is in the middle of a pending Transaction, the\n * Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n * state, new changes are not forwarded to the Subscriber, and catchup\n * is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n * by the Storer, it begins catchup on the Subscriber (with a READONLY\n * snapshot so that it does not block subsequent storage operations).\n * This catchup is thus guaranteed to load the change log entries of\n * that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n * Subscriber from the \"queued\" to the \"active\" set of clients such that\n * the Subscriber begins receiving new changes, starting from the next\n * Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n * is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n * and the Subscriber henceforth forwards changes as they are received.\n \n In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n \n ### Watermarks and ordering\n \n The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n \n https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n \n > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n \n In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n \n A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n \n ```\n * LSN: 1 2 3 4 5 6 7 8 9 10\n * tx1: begin data data data commit\n * tx2: begin data data data commit\n * ```\n \n will arrive as:\n \n ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n \n Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n \n Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n \n * A `commit` message is assigned to a watermark corresponding to its LSN.\n * These are guaranteed to be in commit order by definition.\n \n * `begin` and `data` messages are assigned to the watermark of the\n * preceding `commit` (the previous transaction, or the replication\n * slot's starting LSN) plus 1. This guarantees that they will be sorted\n * after the previously commit transaction even if their LSNs came before it.\n * This is referred to as the `preCommitWatermark`.\n \n * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n * the position of the message within its transaction, with the `begin` message\n * starting at `0`. This guarantees that `begin` and `data` messages will be\n * fetched in the original ChangeSource order during catchup.\n \n `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n \n ### Cleanup\n \n As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n \n More concretely:\n \n * The `initial`, backup-derived watermark is the earliest to which cleanup\n * should ever happen.\n \n * However, it is possible for the replica backup to be ahead of a connected\n * subscriber; and if a network error causes that subscriber to retry from its\n * last watermark, the change streamer must support it.\n \n Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n /\nclass ChangeStreamerImpl implements ChangeStreamerService {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #replicaVersion: string;\n readonly #source: ChangeSource;\n readonly #storer: Storer;\n readonly #forwarder: Forwarder;\n readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n readonly #autoReset: boolean;\n readonly #state: RunningState;\n readonly #initialWatermarks = new Set<string>();\n\n // Starting the (Postgres) ChangeStream results in killing the previous\n // Postgres subscriber, potentially creating a gap in which the old\n // change-streamer has shut down and the new change-streamer has not yet\n // been recognized as \"healthy\" (and thus does not get any requests).\n //\n // To minimize this gap, delay starting the ChangeStream until the first\n // request from a `serving` replicator, indicating that higher level\n // load-balancing / routing logic has begun routing requests to this task.\n readonly #serving = resolver();\n\n readonly #txCounter = getOrCreateCounter(\n 'replication',\n 'transactions',\n 'Count of replicated transactions',\n );\n readonly #changeCounter = getOrCreateCounter(\n 'replication',\n 'changes',\n 'Count of replicated changes (DML or DDL statements)',\n );\n\n #latestStatus: Status;\n #stream: ChangeStream \| undefined;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n replicaVersion: string,\n source: ChangeSource,\n replicationStatusPublisher: ReplicationStatusPublisher,\n autoReset: boolean,\n backPressureLimitHeapProportion: number,\n flowControlConsensusPaddingSeconds: number,\n setTimeoutFn = setTimeout,\n ) {\n this.id = `change-streamer`;\n this.#lc = lc.withContext('component', 'change-streamer');\n this.#shard = shard;\n this.#changeDB = changeDB;\n this.#replicaVersion = replicaVersion;\n this.#source = source;\n this.#storer = new Storer(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n err => this.stop(err),\n backPressureLimitHeapProportion,\n );\n this.#forwarder = new Forwarder(lc, {\n flowControlConsensusPaddingSeconds,\n });\n this.#replicationStatusPublisher = replicationStatusPublisher;\n this.#autoReset = autoReset;\n this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n this.#latestStatus = {tag: 'status'};\n }\n\n async run() {\n this.#lc.info?.('starting change stream');\n\n this.#forwarder.startProgressMonitor();\n\n const lagReport = await this.#source.startLagReporter();\n if (lagReport) {\n this.#latestStatus.lagReport = lagReport;\n }\n\n // Once this change-streamer acquires \"ownership\" of the change DB,\n // it is safe to start the storer.\n await this.#storer.assumeOwnership();\n\n // The threshold in (estimated number of) bytes to send() on subscriber\n // websockets before `await`-ing the I/O buffers to be ready for more.\n const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n while (this.#state.shouldRun()) {\n let err: unknown;\n let watermark: string \| null = null;\n let unflushedBytes = 0;\n try {\n const {lastWatermark, backfillRequests} =\n await this.#storer.getStartStreamInitializationParameters();\n const stream = await this.#source.startStream(\n lastWatermark,\n backfillRequests,\n );\n this.#storer.run().catch(e => stream.changes.cancel(e));\n\n this.#stream = stream;\n if (\n this.#state.resetBackoff() >\n REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n ) {\n // After recovering from a backoff for which a replication status\n // error was published, publish an OK status\n this.#replicationStatusPublisher.publish(\n this.#lc,\n 'Replicating',\n `Replicating from ${lastWatermark}`,\n );\n }\n watermark = null;\n\n for await (const change of stream.changes) {\n const [type, msg] = change;\n switch (type) {\n case 'status':\n if (msg.ack) {\n this.#storer.status(change); // storer acks once it gets through its queue\n }\n if (msg.lagReport) {\n // Lag reports are not stored in the cdc change log, but rather\n // only forwarded on \"live\" connections. When a new subscriber\n // is catching up, it is initialized with the #latestStatus\n // from which it can measure lag while catching up.\n this.#latestStatus.lagReport = msg.lagReport;\n this.#forwarder.sendStatus(this.#latestStatus);\n }\n continue;\n case 'control':\n await this.#handleControlMessage(msg);\n continue; // control messages are not stored/forwarded\n case 'begin':\n watermark = change[2].commitWatermark;\n break;\n case 'commit':\n if (watermark !== change[2].watermark) {\n throw new UnrecoverableError(\n `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n );\n }\n this.#txCounter.add(1);\n break;\n default:\n if (type === 'data') {\n this.#changeCounter.add(1);\n }\n if (watermark === null) {\n throw new UnrecoverableError(\n `${type} change (${msg.tag}) received before 'begin' message`,\n );\n }\n break;\n }\n\n const entry: WatermarkedChange = [watermark, change];\n unflushedBytes += this.#storer.store(entry);\n if (unflushedBytes < flushBytesThreshold) {\n // pipeline changes until flushBytesThreshold\n this.#forwarder.forward(entry);\n } else {\n // Wait for messages to clear socket buffers to ensure that they\n // make their way to subscribers. Without this `await`, the\n // messages end up being buffered in this process, which:\n // (1) results in memory pressure and increased GC activity\n // (2) prevents subscribers from processing the messages as they\n // arrive, instead getting them in a large batch after being\n // idle while they were queued (causing further delays).\n await this.#forwarder.forwardWithFlowControl(entry);\n unflushedBytes = 0;\n }\n\n if (type === 'commit' \|\| type === 'rollback') {\n watermark = null;\n }\n\n // Allow the storer to exert back pressure.\n const readyForMore = this.#storer.readyForMore();\n if (readyForMore) {\n await readyForMore;\n }\n }\n } catch (e) {\n err = e;\n } finally {\n this.#stream?.changes.cancel();\n this.#stream = undefined;\n }\n\n // When the change stream is interrupted, abort any pending transaction.\n if (watermark) {\n this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n this.#storer.abort();\n this.#forwarder.forward([watermark, ['rollback', {tag: 'rollback'}]]);\n }\n\n // Backoff and drain any pending entries in the storer before reconnecting.\n await Promise.all([\n this.#storer.stop(),\n this.#state.backoff(this.#lc, err),\n this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n ? publishCriticalEvent(\n this.#lc,\n replicationStatusError(this.#lc, 'Replicating', err),\n )\n : promiseVoid,\n ]);\n }\n\n this.#forwarder.stopProgressMonitor();\n this.#lc.info?.('ChangeStreamer stopped');\n }\n\n async #handleControlMessage(msg: ChangeStreamControl[1]) {\n this.#lc.info?.('received control message', msg);\n const {tag} = msg;\n\n switch (tag) {\n case 'reset-required':\n await markResetRequired(this.#changeDB, this.#shard);\n await publishReplicationError(\n this.#lc,\n 'Replicating',\n msg.message ?? 'Resync required',\n msg.errorDetails,\n );\n if (this.#autoReset) {\n this.#lc.warn?.('shutting down for auto-reset');\n await this.stop(new AutoResetSignal());\n }\n break;\n default:\n unreachable(tag);\n }\n }\n\n subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n if (mode === 'serving') {\n this.#serving.resolve();\n }\n const downstream = Subscription.create<Downstream>({\n cleanup: () => this.#forwarder.remove(subscriber),\n });\n const subscriber = new Subscriber(\n protocolVersion,\n id,\n watermark,\n downstream,\n () => this.#latestStatus,\n );\n if (replicaVersion !== this.#replicaVersion) {\n this.#lc.warn?.(\n `rejecting subscriber at replica version ${replicaVersion}`,\n );\n subscriber.close(\n ErrorType.WrongReplicaVersion,\n `current replica version is ${\n this.#replicaVersion\n } (requested ${replicaVersion})`,\n );\n } else {\n this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n this.#forwarder.add(subscriber);\n this.#storer.catchup(subscriber, mode);\n }\n return Promise.resolve(downstream);\n }\n\n scheduleCleanup(watermark: string) {\n const origSize = this.#initialWatermarks.size;\n this.#initialWatermarks.add(watermark);\n\n if (origSize === 0) {\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n\n async getChangeLogState(): Promise<{\n replicaVersion: string;\n minWatermark: string;\n }> {\n const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n if (!minWatermark) {\n this.#lc.warn?.(\n `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n );\n }\n return {\n replicaVersion: this.#replicaVersion,\n minWatermark: minWatermark ?? this.#replicaVersion,\n };\n }\n\n /\n Makes a best effort to purge the change log. In the event of a database\n * error, exceptions will be logged and swallowed, so this method is safe\n * to run in a timeout.\n /\n async #purgeOldChanges(): Promise<void> {\n const initial = [...this.#initialWatermarks];\n if (initial.length === 0) {\n this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n return;\n }\n const current = [...this.#forwarder.getAcks()];\n if (current.length === 0) {\n // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n // Bail to be safe.\n this.#lc.warn?.('No subscribers to confirm cleanup');\n return;\n }\n try {\n const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n if (earliestCurrent < earliestInitial) {\n this.#lc.info?.(\n `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n );\n } else {\n this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n const start = performance.now();\n const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n );\n this.#initialWatermarks.delete(earliestInitial);\n }\n } catch (e) {\n this.#lc.warn?.(`error purging change log`, e);\n } finally {\n if (this.#initialWatermarks.size) {\n // If there are unpurged watermarks to check, schedule the next purge.\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n }\n\n async stop(err?: unknown) {\n this.#state.stop(this.#lc, err);\n this.#stream?.changes.cancel();\n await this.#storer.stop();\n await this.#source.stop();\n }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n// `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS 3;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAuDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,iCACA,oCACA,eAAe,YACiB;AAEhC,OAAM,yBAAyB,IAAI,UAAU,MAAM;AACnD,OAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,aACD;CAED,MAAM,EAAC,mBAAkB;AACzB,QAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,iCACA,oCACA,aACD;;AAGH,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwJpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,KAAa;CAU/C,WAAoB,UAAU;CAE9B,aAAsB,mBACpB,eACA,gBACA,mCACD;CACD,iBAA0B,mBACxB,eACA,WACA,sDACD;CAED;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,WACA,iCACA,oCACA,eAAe,YACf;AACA,OAAK,KAAK;AACV,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AACzD,QAAA,QAAc;AACd,QAAA,WAAiB;AACjB,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,SAAe,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,MAAA,QAAc,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;GAAG,CAAC,GACzE,QAAO,KAAK,KAAK,IAAI,EACrB,gCACD;AACD,QAAA,YAAkB,IAAI,UAAU,IAAI,EAClC,oCACD,CAAC;AACF,QAAA,6BAAmC;AACnC,QAAA,YAAkB;AAClB,QAAA,QAAc,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,aAAa;AAChE,QAAA,eAAqB,EAAC,KAAK,UAAS;;CAGtC,MAAM,MAAM;AACV,QAAA,GAAS,OAAO,yBAAyB;AAEzC,QAAA,UAAgB,sBAAsB;EAEtC,MAAM,YAAY,MAAM,MAAA,OAAa,kBAAkB;AACvD,MAAI,UACF,OAAA,aAAmB,YAAY;AAKjC,QAAM,MAAA,OAAa,iBAAiB;EAIpC,MAAM,sBAAsB,wBAAwB,MAAM;AAE1D,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;AACrB,OAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,MAAA,OAAa,wCAAwC;IAC7D,MAAM,SAAS,MAAM,MAAA,OAAa,YAChC,eACA,iBACD;AACD,UAAA,OAAa,KAAK,CAAC,OAAM,MAAK,OAAO,QAAQ,OAAO,EAAE,CAAC;AAEvD,UAAA,SAAe;AACf,QACE,MAAA,MAAY,cAAc,GAC1B,4CAIA,OAAA,2BAAiC,QAC/B,MAAA,IACA,eACA,oBAAoB,gBACrB;AAEH,gBAAY;AAEZ,eAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;AACpB,aAAQ,MAAR;MACE,KAAK;AACH,WAAI,IAAI,IACN,OAAA,OAAa,OAAO,OAAO;AAE7B,WAAI,IAAI,WAAW;AAKjB,cAAA,aAAmB,YAAY,IAAI;AACnC,cAAA,UAAgB,WAAW,MAAA,aAAmB;;AAEhD;MACF,KAAK;AACH,aAAM,MAAA,qBAA2B,IAAI;AACrC;MACF,KAAK;AACH,mBAAY,OAAO,GAAG;AACtB;MACF,KAAK;AACH,WAAI,cAAc,OAAO,GAAG,UAC1B,OAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,YAC7E;AAEH,aAAA,UAAgB,IAAI,EAAE;AACtB;MACF;AACE,WAAI,SAAS,OACX,OAAA,cAAoB,IAAI,EAAE;AAE5B,WAAI,cAAc,KAChB,OAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,mCAC5B;AAEH;;KAGJ,MAAM,QAA2B,CAAC,WAAW,OAAO;AACpD,uBAAkB,MAAA,OAAa,MAAM,MAAM;AAC3C,SAAI,iBAAiB,oBAEnB,OAAA,UAAgB,QAAQ,MAAM;UACzB;AAQL,YAAM,MAAA,UAAgB,uBAAuB,MAAM;AACnD,uBAAiB;;AAGnB,SAAI,SAAS,YAAY,SAAS,WAChC,aAAY;KAId,MAAM,eAAe,MAAA,OAAa,cAAc;AAChD,SAAI,aACF,OAAM;;YAGH,GAAG;AACV,UAAM;aACE;AACR,UAAA,QAAc,QAAQ,QAAQ;AAC9B,UAAA,SAAe,KAAA;;AAIjB,OAAI,WAAW;AACb,UAAA,GAAS,OAAO,oCAAoC,YAAY;AAChE,UAAA,OAAa,OAAO;AACpB,UAAA,UAAgB,QAAQ,CAAC,WAAW,CAAC,YAAY,EAAC,KAAK,YAAW,CAAC,CAAC,CAAC;;AAIvE,SAAM,QAAQ,IAAI;IAChB,MAAA,OAAa,MAAM;IACnB,MAAA,MAAY,QAAQ,MAAA,IAAU,IAAI;IAClC,MAAA,MAAY,aAAa,8CACrB,qBACE,MAAA,IACA,uBAAuB,MAAA,IAAU,eAAe,IAAI,CACrD,GACD;IACL,CAAC;;AAGJ,QAAA,UAAgB,qBAAqB;AACrC,QAAA,GAAS,OAAO,yBAAyB;;CAG3C,OAAA,qBAA4B,KAA6B;AACvD,QAAA,GAAS,OAAO,4BAA4B,IAAI;EAChD,MAAM,EAAC,QAAO;AAEd,UAAQ,KAAR;GACE,KAAK;AACH,UAAM,kBAAkB,MAAA,UAAgB,MAAA,MAAY;AACpD,UAAM,wBACJ,MAAA,IACA,eACA,IAAI,WAAW,mBACf,IAAI,aACL;AACD,QAAI,MAAA,WAAiB;AACnB,WAAA,GAAS,OAAO,+BAA+B;AAC/C,WAAM,KAAK,KAAK,IAAI,iBAAiB,CAAC;;AAExC;GACF,QACE,aAAY,IAAI;;;CAItB,UAAU,KAAqD;EAC7D,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;AAC/D,MAAI,SAAS,UACX,OAAA,QAAc,SAAS;EAEzB,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe,MAAA,UAAgB,OAAO,WAAW,EAClD,CAAC;EACF,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,MAAA,aACP;AACD,MAAI,mBAAmB,MAAA,gBAAsB;AAC3C,SAAA,GAAS,OACP,2CAA2C,iBAC5C;AACD,cAAW,MACT,GACA,8BACE,MAAA,eACD,cAAc,eAAe,GAC/B;SACI;AACL,SAAA,GAAS,QAAQ,qBAAqB,WAAW,KAAK;AAEtD,SAAA,UAAgB,IAAI,WAAW;AAC/B,SAAA,OAAa,QAAQ,YAAY,KAAK;;AAExC,SAAO,QAAQ,QAAQ,WAAW;;CAGpC,gBAAgB,WAAmB;EACjC,MAAM,WAAW,MAAA,kBAAwB;AACzC,QAAA,kBAAwB,IAAI,UAAU;AAEtC,MAAI,aAAa,EACf,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;CAI3E,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,MAAA,OAAa,2BAA2B;AACnE,MAAI,CAAC,aACH,OAAA,GAAS,OACP,+EACD;AAEH,SAAO;GACL,gBAAgB,MAAA;GAChB,cAAc,gBAAgB,MAAA;GAC/B;;;;;;;CAQH,OAAA,kBAAwC;EACtC,MAAM,UAAU,CAAC,GAAG,MAAA,kBAAwB;AAC5C,MAAI,QAAQ,WAAW,GAAG;AACxB,SAAA,GAAS,OAAO,6CAA6C;AAC7D;;EAEF,MAAM,UAAU,CAAC,GAAG,MAAA,UAAgB,SAAS,CAAC;AAC9C,MAAI,QAAQ,WAAW,GAAG;AAGxB,SAAA,GAAS,OAAO,oCAAoC;AACpD;;AAEF,MAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,QAAoC;GACpE,MAAM,kBAAkB,IAAI,GAAI,QAAoC;AACpE,OAAI,kBAAkB,gBACpB,OAAA,GAAS,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,GAC/E;QACI;AACL,UAAA,GAAS,OAAO,0BAA0B,gBAAgB,MAAM;IAChE,MAAM,QAAQ,YAAY,KAAK;IAC/B,MAAM,UAAU,MAAM,MAAA,OAAa,mBAAmB,gBAAgB;IACtE,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,UAAA,GAAS,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,MACjE;AACD,UAAA,kBAAwB,OAAO,gBAAgB;;WAE1C,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;YACtC;AACR,OAAI,MAAA,kBAAwB,KAE1B,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;;CAK7E,MAAM,KAAK,KAAe;AACxB,QAAA,MAAY,KAAK,MAAA,IAAU,IAAI;AAC/B,QAAA,QAAc,QAAQ,QAAQ;AAC9B,QAAM,MAAA,OAAa,MAAM;AACzB,QAAM,MAAA,OAAa,MAAM;;;AAmB7B,IAAM,mBAAmB,6BAA6B"}

package/out/zero-cache/src/services/life-cycle.d.ts CHANGED Viewed

@@ -11,13 +11,14 @@ import type { SingletonService } from './service.ts';
  *   the `SIGTERM` signal only after all `user-facing` workers have
  *   exited.
  *
- * For other kill signals, such as `SIGQUIT`, all workers
- * are stopped without draining. Additionally, if any worker exits
- * unexpectedly, all workers sent an immediate `SIGQUIT` signal.
+ * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers
+ * are stopped without draining. `SIGQUIT` is used to represent an
+ * intentional shutdown (for which draining is not beneficial), whereas
+ * `SIGABRT` is used for unexpected process exits.
  */
 export type WorkerType = 'user-facing' | 'supporting';
 export declare const GRACEFUL_SHUTDOWN: readonly ["SIGTERM", "SIGINT"];
-export declare const FORCEFUL_SHUTDOWN: readonly ["SIGQUIT"];
+export declare const FORCEFUL_SHUTDOWN: readonly ["SIGQUIT", "SIGABRT"];
 export declare const UNHANDLED_EXCEPTION_ERROR_CODE = 13;
 /**
  * Handles readiness, termination signals, and coordination of graceful

package/out/zero-cache/src/services/life-cycle.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD~~;;;;;;;;;;;GAWG~~;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,gCAAiC,CAAC;AAChE,eAAO,MAAM,iBAAiB,~~sBAAuB~~,CAAC;~~AAKtD~~,eAAO,MAAM,8BAA8B,KAAK,CAAC;~~AAEjD~~;;;GAGG;AACH,qBAAa,cAAc;;gBAWb,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY;~~IAiC9C~~,IAAI;IAoBJ,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;IA2B9D,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAiBjE,YAAY,IAAI,MAAM,EAAE;IAIlB,eAAe;IAIrB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;~~CAwE3C~~;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAWvD;AAID;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;gBAQf,EAAE,EAAE,UAAU,EAAE,YAAY,SAA2B;IAKnE,WAAW,CAAC,UAAU,EAAE,mBAAmB;IAsD3C,IAAI;CAML"}
1	+ {"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,gCAAiC,CAAC;AAChE,eAAO,MAAM,iBAAiB,iCAAkC,CAAC;AAOjE,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAOjD;;;GAGG;AACH,qBAAa,cAAc;;gBAWb,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY;IA2C9C,IAAI;IAoBJ,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;IA2B9D,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAiBjE,YAAY,IAAI,MAAM,EAAE;IAIlB,eAAe;IAIrB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;CAiF3C;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAWvD;AAID;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;gBAQf,EAAE,EAAE,UAAU,EAAE,YAAY,SAA2B;IAKnE,WAAW,CAAC,UAAU,EAAE,mBAAmB;IAsD3C,IAAI;CAML"}