npm - @rocicorp/zero - Versions diffs - 1.6.0-canary.1 → 1.6.0-canary.10 - Mend

@rocicorp/zero 1.6.0-canary.1 → 1.6.0-canary.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (1772) hide show

package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"initial-sync.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"sourcesContent":["import {mkdtemp, rm} from 'node:fs/promises';\nimport {platform, tmpdir} from 'node:os';\nimport {join} from 'node:path';\nimport {Writable} from 'node:stream';\nimport {pipeline} from 'node:stream/promises';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport type {DownloadStatus} from '../../../../../zero-events/src/status.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {\n createLiteIndexStatement,\n createLiteTableStatement,\n} from '../../../db/create.ts';\nimport {\n computeZqlSpecs,\n listIndexes,\n listTables,\n} from '../../../db/lite-tables.ts';\nimport * as Mode from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {\n mapPostgresToLite,\n mapPostgresToLiteIndex,\n} from '../../../db/pg-to-lite.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport type {IndexSpec, PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {\n JSON_STRINGIFIED,\n liteValue,\n type LiteValueType,\n} from '../../../types/lite.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport {PG_15, PG_17} from '../../../types/pg-versions.ts';\nimport {\n pgClient,\n type PostgresDB,\n type PostgresTransaction,\n type PostgresValueType,\n} from '../../../types/pg.ts';\nimport {CpuProfiler} from '../../../types/profiler.ts';\nimport type {ShardConfig} from '../../../types/shards.ts';\nimport {ALLOWED_APP_ID_CHARACTERS} from '../../../types/shards.ts';\nimport {id} from '../../../types/sql.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {ColumnMetadataStore} from '../../replicator/schema/column-metadata.ts';\nimport {initReplicationState} from '../../replicator/schema/replication-state.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicaAndSlot} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport {\n dropShard,\n getInternalShardConfig,\n initReplica,\n validatePublications,\n} from './schema/shard.ts';\n\nexport type InitialSyncOptions = {\n tableCopyWorkers: number;\n profileCopy?: boolean | undefined;\n textCopy?: boolean | undefined;\n replicationSlotFailover?: boolean | undefined;\n /**\n * When set, run initial sync in \"shadow\" mode for verification: skip all\n * upstream mutations (no replication slot, no addReplica, no dropShard, no\n * slot drop on failure), suppress status events, and optionally sample\n * rows from each table via TABLESAMPLE BERNOULLI + LIMIT. The caller is\n * responsible for providing (and discarding) a throwaway SQLite `tx`.\n */\n shadow?:\n | {\n /** 0 < rate <= 1. When 1, no TABLESAMPLE clause is added. */\n sampleRate: number;\n /**\n * LIMIT N cap appended after TABLESAMPLE. Required: shadow sync is\n * for verification only, so every run must commit to a row budget.\n */\n maxRowsPerTable: number;\n }\n | undefined;\n};\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n syncOptions: InitialSyncOptions,\n context: ServerContext,\n) {\n if (!ALLOWED_APP_ID_CHARACTERS.test(shard.appID)) {\n throw new Error(\n 'The App ID may only consist of lower-case letters, numbers, and the underscore character',\n );\n }\n const {\n tableCopyWorkers,\n profileCopy,\n textCopy = false,\n replicationSlotFailover = false,\n shadow,\n } = syncOptions;\n const copyProfiler = profileCopy ? await CpuProfiler.connect() : null;\n const sql = pgClient(lc, upstreamURI, 'initial-sync');\n // Replication session is only needed to create a replication slot in the\n // real path. In shadow mode we export a snapshot on a normal connection\n // instead, so no replication session is opened.\n const replicationSession = shadow\n ? undefined\n : pgClient(lc, upstreamURI, 'initial-sync-replication-session', {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n });\n\n const replicaID = Date.now().toString();\n let slotName: string | undefined; // undefined === shadow\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(\n tx,\n shadow ? async () => {} : undefined,\n ).publish(lc, 'Initializing');\n let releaseShadowSnapshot: (() => Promise<void>) | undefined;\n try {\n const pgVersion = await checkUpstreamConfig(sql);\n\n // In shadow mode we assume the shard is already initialized and just\n // read back the existing publications. `ensurePublishedTables` would\n // otherwise run DDL and potentially call `dropShard`, which must never\n // happen during a shadow run.\n const {publications} = shadow\n ? await getInternalShardConfig(sql, shard)\n : await ensurePublishedTables(lc, sql, shard);\n lc.info?.(`Upstream is setup with publications [${publications}]`);\n\n const {database, host} = sql.options;\n lc.info?.(\n shadow\n ? `acquiring exported snapshot on ${database}@${host} (shadow mode)`\n : `opening replication session to ${database}@${host}`,\n );\n\n let snapshot: string;\n let lsn: string;\n\n if (shadow) {\n const acquired = await acquireExportedSnapshotForShadowSync(\n lc,\n upstreamURI,\n );\n snapshot = acquired.snapshot;\n lsn = acquired.lsn;\n releaseShadowSnapshot = acquired.release;\n } else {\n const slot = await createReplicaAndSlot(\n lc,\n sql,\n must(replicationSession),\n shard,\n replicaID,\n replicationSlotFailover && pgVersion >= PG_17,\n );\n snapshot = slot.snapshot_name;\n lsn = slot.consistent_point;\n slotName = slot.slot_name;\n }\n\n const initialVersion = toStateVersionString(lsn);\n\n initReplicationState(tx, publications, initialVersion, context);\n\n // Run up to MAX_WORKERS to copy of tables at the replication slot's snapshot.\n const start = performance.now();\n // Retrieve the published schema at the consistent_point.\n const published = await runTx(\n sql,\n async tx => {\n await tx.unsafe(/* sql*/ `SET TRANSACTION SNAPSHOT '${snapshot}'`);\n return getPublicationInfo(tx, publications);\n },\n {mode: Mode.READONLY},\n );\n // Note: If this throws, initial-sync is aborted.\n validatePublications(lc, published);\n\n // Now that tables have been validated, kick off the copiers.\n const {tables, indexes} = published;\n const numTables = tables.length;\n if (platform() === 'win32' && tableCopyWorkers < numTables) {\n lc.warn?.(\n `Increasing the number of copy workers from ${tableCopyWorkers} to ` +\n `${numTables} to work around a Node/Postgres connection bug`,\n );\n }\n const numWorkers =\n platform() === 'win32'\n ? numTables\n : Math.min(tableCopyWorkers, numTables);\n\n const copyPool = pgClient(lc, upstreamURI, 'initial-sync-copy-worker', {\n max: numWorkers,\n ['max_lifetime']: 120 * 60, // set a long (2h) limit for COPY streaming\n });\n const copiers = startTableCopyWorkers(\n lc,\n copyPool,\n snapshot,\n numWorkers,\n numTables,\n );\n try {\n createLiteTables(tx, tables, initialVersion);\n const sampleRate = shadow?.sampleRate;\n const maxRowsPerTable = shadow?.maxRowsPerTable;\n const downloads = await Promise.all(\n tables.map(spec =>\n copiers.processReadTask((db, lc) =>\n getInitialDownloadState(lc, db, spec, shadow !== undefined),\n ),\n ),\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying ${numTables} upstream tables at version ${initialVersion}`,\n 5000,\n () => ({downloadStatus: downloads.map(({status}) => status)}),\n );\n\n void copyProfiler?.start();\n const rowCounts = await Promise.all(\n downloads.map(table =>\n copiers.processReadTask((db, lc) =>\n copy(\n lc,\n table,\n copyPool,\n db,\n tx,\n textCopy,\n sampleRate,\n maxRowsPerTable,\n ),\n ),\n ),\n );\n void copyProfiler?.stopAndDispose(lc, 'initial-copy');\n copiers.setDone();\n\n const total = rowCounts.reduce(\n (acc, curr) => ({\n rows: acc.rows + curr.rows,\n flushTime: acc.flushTime + curr.flushTime,\n }),\n {rows: 0, flushTime: 0},\n );\n\n statusPublisher.publish(\n lc,\n 'Indexing',\n `Creating ${indexes.length} indexes`,\n 5000,\n );\n const indexStart = performance.now();\n createLiteIndices(tx, indexes);\n const index = performance.now() - indexStart;\n lc.info?.(`Created indexes (${index.toFixed(3)} ms)`);\n\n if (slotName && replicaID) {\n await initReplica(sql, shard, replicaID, published, context);\n } else {\n assert(shadow, 'expected to be in shadow sync if there is no slotName');\n const rowsByTable = new Map<string, number>();\n for (let i = 0; i < downloads.length; i++) {\n rowsByTable.set(downloads[i].status.table, rowCounts[i].rows);\n }\n verifyShadowReplica(lc, tx, published, rowsByTable);\n }\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Synced ${total.rows.toLocaleString()} rows of ${numTables} tables in ${publications} up to ${lsn} ` +\n `(flush: ${total.flushTime.toFixed(3)}, index: ${index.toFixed(3)}, total: ${elapsed.toFixed(3)} ms)`,\n );\n } finally {\n // All meaningful errors are handled at the processReadTask() call site.\n void copyPool.end().catch(e => lc.warn?.(`Error closing copyPool`, e));\n }\n } catch (e) {\n if (slotName) {\n // If initial-sync did not succeed, make a best effort to drop the\n // orphaned replication slot to avoid running out of slots in\n // pathological cases that result in repeated failures.\n lc.warn?.(`dropping replication slot ${slotName}`, e);\n await sql`\n SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n WHERE slot_name = ${slotName};\n `.catch(e => lc.warn?.(`Unable to drop replication slot ${slotName}`, e));\n }\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n if (releaseShadowSnapshot) {\n await releaseShadowSnapshot().catch(e =>\n lc.warn?.(`Error releasing shadow snapshot`, e),\n );\n }\n if (replicationSession) {\n await replicationSession.end();\n }\n await sql.end();\n }\n}\n\nexport type ShadowSyncOptions = {\n sampleRate: number;\n maxRowsPerTable: number;\n /**\n * Parent directory for the throwaway SQLite replica. Defaults to the OS\n * tmpdir. Primarily for tests that need to isolate the scratch directory.\n */\n parentDir?: string | undefined;\n};\n\n/**\n * Exercises the initial-sync code path against a sample of rows from every\n * published table, writing into a throwaway SQLite database that is deleted\n * when the run ends. Produces zero upstream mutations: no replication slot,\n * no `addReplica`, no `dropShard`, no status events.\n *\n * Intended to be invoked periodically so that if a customer ever needs a\n * full reset, we have recent confidence that `initialSync` still works.\n * The shard must already be initialized upstream.\n */\nexport async function shadowInitialSync(\n lc: LogContext,\n shard: ShardConfig,\n upstreamURI: string,\n shadow: ShadowSyncOptions,\n context: ServerContext,\n syncOptions?: Pick<InitialSyncOptions, 'textCopy'>,\n): Promise<void> {\n const dir = await mkdtemp(\n join(shadow.parentDir ?? tmpdir(), 'zero-shadow-sync-'),\n );\n const dbPath = join(dir, 'shadow-replica.db');\n const db = new Database(lc, dbPath);\n try {\n await initialSync(\n lc,\n shard,\n db,\n upstreamURI,\n {\n // Shadow sync copies small samples, so one worker is plenty —\n // no reason to burn additional upstream connections.\n tableCopyWorkers: 1,\n textCopy: syncOptions?.textCopy,\n shadow,\n },\n context,\n );\n } finally {\n try {\n db.close();\n } catch (e) {\n lc.warn?.(`Error closing shadow replica db`, e);\n }\n await rm(dir, {recursive: true, force: true}).catch(e =>\n lc.warn?.(`Error cleaning up shadow replica dir ${dir}`, e),\n );\n }\n}\n\nasync function checkUpstreamConfig(sql: PostgresDB) {\n const {walLevel, version} = (\n await sql<{walLevel: string; version: number}[]>`\n SELECT current_setting('wal_level') as \"walLevel\", \n current_setting('server_version_num') as \"version\";\n `\n )[0];\n\n if (walLevel !== 'logical') {\n throw new Error(\n `Postgres must be configured with \"wal_level = logical\" (currently: \"${walLevel})`,\n );\n }\n if (version < PG_15) {\n throw new Error(\n `Must be running Postgres 15 or higher (currently: \"${version}\")`,\n );\n }\n return version;\n}\n\nasync function ensurePublishedTables(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardConfig,\n validate = true,\n): Promise<{publications: string[]}> {\n const {database, host} = sql.options;\n lc.info?.(`Ensuring upstream PUBLICATION on ${database}@${host}`);\n\n await ensureShardSchema(lc, sql, shard);\n const {publications} = await getInternalShardConfig(sql, shard);\n\n if (validate) {\n let valid = false;\n const nonInternalPublications = publications.filter(\n p => !p.startsWith('_'),\n );\n const exists = await sql`\n SELECT pubname FROM pg_publication WHERE pubname IN ${sql(publications)}\n `.values();\n if (exists.length !== publications.length) {\n lc.warn?.(\n `some configured publications [${publications}] are missing: ` +\n `[${exists.flat()}]. resyncing`,\n );\n } else if (\n !equals(new Set(shard.publications), new Set(nonInternalPublications))\n ) {\n lc.warn?.(\n `requested publications [${shard.publications}] differ from previous` +\n `publications [${nonInternalPublications}]. resyncing`,\n );\n } else {\n valid = true;\n }\n if (!valid) {\n await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n return ensurePublishedTables(lc, sql, shard, false);\n }\n }\n return {publications};\n}\n\nfunction startTableCopyWorkers(\n lc: LogContext,\n db: PostgresDB,\n snapshot: string,\n numWorkers: number,\n numTables: number,\n): TransactionPool {\n const {init} = importSnapshot(snapshot);\n const tableCopiers = new TransactionPool(lc, {\n mode: Mode.READONLY,\n init,\n initialWorkers: numWorkers,\n });\n tableCopiers.run(db);\n\n lc.info?.(`Started ${numWorkers} workers to copy ${numTables} tables`);\n\n if (parseInt(process.versions.node) < 22) {\n lc.warn?.(\n `\\n\\n\\n` +\n `Older versions of Node have a bug that results in an unresponsive\\n` +\n `Postgres connection after running certain combinations of COPY commands.\\n` +\n `If initial sync hangs, run zero-cache with Node v22+. This has the additional\\n` +\n `benefit of being consistent with the Node version run in the production container image.` +\n `\\n\\n\\n`,\n );\n }\n return tableCopiers;\n}\n\n/**\n * Shadow-mode alternative to `createReplicationSlot`: opens a dedicated\n * READ ONLY REPEATABLE READ transaction on a normal connection, exports the\n * snapshot and captures the current WAL LSN, then holds the transaction\n * open until `release()` is called. The held transaction keeps the snapshot\n * importable by the table-copy workers for the duration of the COPY phase.\n *\n * Idle-in-transaction timeout is disabled locally so the exporter doesn't\n * get killed while workers are still importing.\n */\nasync function acquireExportedSnapshotForShadowSync(\n lc: LogContext,\n upstreamURI: string,\n): Promise<{\n snapshot: string;\n lsn: string;\n release: () => Promise<void>;\n}> {\n const holder = pgClient(lc, upstreamURI, 'shadow-initial-sync-snapshot', {\n max: 1,\n });\n const ready = resolver<{snapshot: string; lsn: string}>();\n const release = resolver<void>();\n const held = holder\n .begin(Mode.READONLY, async tx => {\n await tx`SET LOCAL idle_in_transaction_session_timeout = 0`.execute();\n const [row] = await tx<{snapshot: string; lsn: string}[]>`\n SELECT pg_export_snapshot() AS snapshot,\n pg_current_wal_lsn()::text AS lsn`;\n ready.resolve(row);\n await release.promise;\n })\n .catch(e => ready.reject(e));\n\n let snapshot: string;\n let lsn: string;\n try {\n ({snapshot, lsn} = await ready.promise);\n } catch (e) {\n await holder\n .end()\n .catch(err =>\n lc.warn?.(`Error ending shadow snapshot holder after failure`, err),\n );\n throw e;\n }\n lc.info?.(\n `Exported snapshot ${snapshot} at LSN ${lsn} (shadow initial sync)`,\n );\n return {\n snapshot,\n lsn,\n release: async () => {\n release.resolve();\n try {\n await held;\n } catch (e) {\n lc.warn?.(`snapshot holder transaction ended with error`, e);\n }\n await holder.end();\n },\n };\n}\n\nfunction createLiteTables(\n tx: Database,\n tables: PublishedTableSpec[],\n initialVersion: string,\n) {\n // TODO: Figure out how to reuse the ChangeProcessor here to avoid\n // duplicating the ColumnMetadata logic.\n const columnMetadata = must(ColumnMetadataStore.getInstance(tx));\n for (const t of tables) {\n tx.exec(createLiteTableStatement(mapPostgresToLite(t, initialVersion)));\n const tableName = liteTableName(t);\n for (const [colName, colSpec] of Object.entries(t.columns)) {\n columnMetadata.insert(tableName, colName, colSpec);\n }\n }\n}\n\nfunction createLiteIndices(tx: Database, indices: IndexSpec[]) {\n for (const index of indices) {\n tx.exec(createLiteIndexStatement(mapPostgresToLiteIndex(index)));\n }\n}\n\n/**\n * Runs structural assertions over a just-synced replica and throws if any\n * fail. Only called in shadow mode — a successful return means the replica\n * is schema-complete, row-count consistent, ZQL-queryable, and its column\n * metadata is in sync with its lite schema.\n *\n * Exported for testing.\n */\nexport function verifyShadowReplica(\n lc: LogContext,\n db: Database,\n published: {tables: PublishedTableSpec[]; indexes: IndexSpec[]},\n rowsByTable: ReadonlyMap<string, number>,\n): void {\n const issues: string[] = [];\n let columnsChecked = 0;\n let rowsChecked = 0;\n\n // 1. Schema completeness: every published table exists in the replica\n // with at least the expected column set.\n const liteTables = listTables(db);\n const liteTableByName = new Map(liteTables.map(t => [t.name, t]));\n for (const pt of published.tables) {\n const name = liteTableName(pt);\n const lite = liteTableByName.get(name);\n if (!lite) {\n issues.push(`missing table in replica: ${name}`);\n continue;\n }\n for (const col of Object.keys(pt.columns)) {\n columnsChecked++;\n if (!(col in lite.columns)) {\n issues.push(`column missing in replica table ${name}: ${col}`);\n }\n }\n }\n\n // Every published index exists in the replica.\n const liteIndexNames = new Set(listIndexes(db).map(i => i.name));\n for (const ix of published.indexes) {\n const mapped = mapPostgresToLiteIndex(ix);\n if (!liteIndexNames.has(mapped.name)) {\n issues.push(\n `missing index in replica: ${mapped.name} on ${mapped.tableName}`,\n );\n }\n }\n\n // 2. Row counts: SQLite COUNT(*) matches the in-memory copy counter.\n for (const [table, expected] of rowsByTable) {\n try {\n const [row] = db\n .prepare(`SELECT COUNT(*) as count FROM \"${table}\"`)\n .all<{count: number}>();\n if (row.count !== expected) {\n issues.push(\n `row count mismatch for table ${table}: ` +\n `copy counter reported ${expected}, replica has ${row.count}`,\n );\n } else {\n rowsChecked += row.count;\n }\n } catch (e) {\n issues.push(`could not count rows in table ${table}: ${String(e)}`);\n }\n }\n\n // 3. ZQL-queryability: every published table survives computeZqlSpecs's\n // filtering (primary-key candidate, ZQL-typed columns, etc.).\n const tableSpecs = computeZqlSpecs(lc, db, {\n includeBackfillingColumns: false,\n });\n for (const pt of published.tables) {\n const name = liteTableName(pt);\n if (!tableSpecs.has(name)) {\n issues.push(\n `table not queryable via ZQL (dropped by computeZqlSpecs): ${name}`,\n );\n }\n }\n\n // 4. Column metadata: every published column has a _zero.column_metadata row.\n const meta = must(ColumnMetadataStore.getInstance(db));\n for (const pt of published.tables) {\n const name = liteTableName(pt);\n const rows = meta.getTable(name);\n for (const col of Object.keys(pt.columns)) {\n if (!rows.has(col)) {\n issues.push(`missing column_metadata row for ${name}.${col}`);\n }\n }\n }\n\n if (issues.length) {\n throw new Error(\n `Shadow replica verification failed (${issues.length} issue(s)):\\n` +\n issues.map(i => ` - ${i}`).join('\\n'),\n );\n }\n\n lc.info?.(\n `Shadow replica verification passed: ` +\n `${published.tables.length} tables, ` +\n `${published.indexes.length} indexes, ` +\n `${columnsChecked} columns, ` +\n `${rowsChecked.toLocaleString()} rows`,\n );\n}\n\n// Verified empirically that batches of 50 seem to be the sweet spot,\n// similar to the report in https://sqlite.org/forum/forumpost/8878a512d3652655\n//\n// Exported for testing.\nexport const INSERT_BATCH_SIZE = 50;\n\nconst MB = 1024 * 1024;\nconst MAX_BUFFERED_ROWS = 10_000;\nconst BUFFERED_SIZE_THRESHOLD = 8 * MB;\n\nexport type DownloadStatements = {\n select: string;\n getTotalRows: string;\n getTotalBytes: string;\n};\n\n/**\n * Produces ` TABLESAMPLE BERNOULLI(n)` when `sampleRate` is < 1, else `''`.\n * Row-level Bernoulli sampling is used (rather than SYSTEM) because it\n * produces a more uniform sample and, unlike SYSTEM, still returns rows\n * for small tables at low rates.\n */\nfunction tableSampleClause(sampleRate: number | undefined): string {\n if (sampleRate === undefined || sampleRate >= 1) {\n return '';\n }\n // Round away float noise (e.g. 0.3 * 100 = 30.000000000000004) while still\n // preserving sub-integer rates like 0.001 (= 0.1%).\n const pct = parseFloat((sampleRate * 100).toFixed(6));\n return /*sql*/ ` TABLESAMPLE BERNOULLI(${pct})`;\n}\n\nfunction limitClause(maxRowsPerTable: number | undefined): string {\n return maxRowsPerTable !== undefined\n ? /*sql*/ ` LIMIT ${maxRowsPerTable}`\n : '';\n}\n\n/**\n * Returns the SELECT column expressions for binary COPY, casting columns\n * without a known binary decoder to `::text`.\n */\nexport function makeBinarySelectExprs(\n table: PublishedTableSpec,\n cols: string[],\n): string[] {\n return cols.map(col => {\n const spec = table.columns[col];\n return hasBinaryDecoder(spec) ? id(col) : `${id(col)}::text`;\n });\n}\n\nexport function makeDownloadStatements(\n table: PublishedTableSpec,\n cols: string[],\n sampleRate?: number | undefined,\n maxRowsPerTable?: number | undefined,\n selectExprs?: string[] | undefined,\n): DownloadStatements {\n const filterConditions = Object.values(table.publications)\n .map(({rowFilter}) => rowFilter)\n .filter(f => !!f); // remove nulls\n const where =\n filterConditions.length === 0\n ? ''\n : /*sql*/ `WHERE ${filterConditions.join(' OR ')}`;\n const sample = tableSampleClause(sampleRate);\n const limit = limitClause(maxRowsPerTable);\n const fromTable = /*sql*/ `FROM ${id(table.schema)}.${id(table.name)}${sample} ${where}`;\n const select = /*sql*/ `SELECT ${(selectExprs ?? cols.map(id)).join(',')} ${fromTable}${limit}`;\n if (limit) {\n // With LIMIT, wrap counts/sums in a subquery so they reflect the\n // capped rowset rather than the full (sampled) table.\n const bytesExpr = cols\n .map(col => `COALESCE(pg_column_size(${id(col)}), 0)`)\n .join(' + ');\n return {\n select,\n getTotalRows: /*sql*/ `SELECT COUNT(*)::bigint AS \"totalRows\" FROM (SELECT 1 AS _ ${fromTable}${limit}) s`,\n getTotalBytes: /*sql*/ `SELECT COALESCE(SUM(b), 0)::bigint AS \"totalBytes\" FROM (SELECT (${bytesExpr}) AS b ${fromTable}${limit}) s`,\n };\n }\n const totalBytes = `(${cols.map(col => `SUM(COALESCE(pg_column_size(${id(col)}), 0))`).join(' + ')})`;\n return {\n select,\n getTotalRows: /*sql*/ `SELECT COUNT(*) AS \"totalRows\" ${fromTable}`,\n getTotalBytes: /*sql*/ `SELECT ${totalBytes} AS \"totalBytes\" ${fromTable}`,\n };\n}\n\ntype DownloadState = {\n spec: PublishedTableSpec;\n status: DownloadStatus;\n};\n\n// Exported for testing.\nexport async function getInitialDownloadState(\n lc: LogContext,\n sql: PostgresDB,\n spec: PublishedTableSpec,\n skipTotals: boolean,\n): Promise<DownloadState> {\n const start = performance.now();\n const table = liteTableName(spec);\n const columns = Object.keys(spec.columns);\n if (skipTotals) {\n // Shadow sync suppresses status events, so the pg_class\n // estimates would be computed and thrown away.\n return {\n spec,\n status: {table, columns, rows: 0, totalRows: 0, totalBytes: 0},\n };\n }\n // Use pg_class estimates instead of expensive COUNT(*) and\n // SUM(pg_column_size(...)) full table scans. The exact values are only\n // used for progress reporting, so estimates are sufficient.\n const qualifiedName = `${id(spec.schema)}.${id(spec.name)}`;\n const estimateResult = await sql<\n {totalRows: number; totalBytes: number}[]\n >`SELECT GREATEST(reltuples, 0)::float8 AS \"totalRows\",\n pg_table_size(oid)::float8 AS \"totalBytes\"\n FROM pg_class\n WHERE oid = ${qualifiedName}::regclass`;\n\n const {totalRows, totalBytes} = estimateResult[0] ?? {\n totalRows: 0,\n totalBytes: 0,\n };\n\n const state: DownloadState = {\n spec,\n status: {\n table,\n columns,\n rows: 0,\n totalRows,\n totalBytes,\n },\n };\n const elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(`Computed initial download state for ${table} (${elapsed} ms)`, {\n state: state.status,\n });\n return state;\n}\n\nfunction copy(\n lc: LogContext,\n {spec: table, status}: DownloadState,\n dbClient: PostgresDB,\n from: PostgresTransaction,\n to: Database,\n textCopy: boolean,\n sampleRate?: number | undefined,\n maxRowsPerTable?: number | undefined,\n) {\n if (textCopy) {\n return copyText(\n lc,\n table,\n status,\n dbClient,\n from,\n to,\n sampleRate,\n maxRowsPerTable,\n );\n }\n return copyBinary(lc, table, status, from, to, sampleRate, maxRowsPerTable);\n}\n\nasync function copyBinary(\n lc: LogContext,\n table: PublishedTableSpec,\n status: DownloadStatus,\n from: PostgresTransaction,\n to: Database,\n sampleRate?: number | undefined,\n maxRowsPerTable?: number | undefined,\n) {\n const start = performance.now();\n let flushTime = 0;\n\n const tableName = liteTableName(table);\n const orderedColumns = Object.entries(table.columns);\n\n const columnNames = orderedColumns.map(([c]) => c);\n const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n const valuesSql =\n columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n const insertSql = /*sql*/ `\n INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n const insertStmt = to.prepare(insertSql);\n const insertBatchStmt = to.prepare(\n insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n );\n\n // Build SELECT with ::text casts for columns without a known binary decoder.\n const select = makeDownloadStatements(\n table,\n columnNames,\n sampleRate,\n maxRowsPerTable,\n makeBinarySelectExprs(table, columnNames),\n ).select;\n\n const decoders = orderedColumns.map(([, spec]) =>\n hasBinaryDecoder(spec) ? makeBinaryDecoder(spec) : textCastDecoder,\n );\n\n const valuesPerRow = columnSpecs.length;\n const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;\n\n const pendingValues: LiteValueType[] = Array.from({\n length: MAX_BUFFERED_ROWS * valuesPerRow,\n });\n let pendingRows = 0;\n let pendingSize = 0;\n\n function flush() {\n const start = performance.now();\n const flushedRows = pendingRows;\n const flushedSize = pendingSize;\n\n let l = 0;\n for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n }\n for (; pendingRows > 0; pendingRows--) {\n insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n }\n const flushedValues = flushedRows * valuesPerRow;\n for (let i = 0; i < flushedValues; i++) {\n pendingValues[i] = undefined as unknown as LiteValueType;\n }\n pendingSize = 0;\n status.rows += flushedRows;\n\n const elapsed = performance.now() - start;\n flushTime += elapsed;\n lc.debug?.(\n `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n );\n }\n\n const binaryParser = new BinaryCopyParser();\n let col = 0;\n\n lc.info?.(`Starting binary copy stream of ${tableName}:`, select);\n\n await pipeline(\n await from\n .unsafe(`COPY (${select}) TO STDOUT WITH (FORMAT binary)`)\n .readable(),\n new Writable({\n highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n write(\n chunk: Buffer,\n _encoding: string,\n callback: (error?: Error) => void,\n ) {\n try {\n for (const fieldBuf of binaryParser.parse(chunk)) {\n pendingSize += fieldBuf === null ? 4 : fieldBuf.length;\n pendingValues[pendingRows * valuesPerRow + col] =\n fieldBuf === null ? null : decoders[col](fieldBuf);\n\n if (++col === decoders.length) {\n col = 0;\n if (\n ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow ||\n pendingSize >= BUFFERED_SIZE_THRESHOLD\n ) {\n flush();\n }\n }\n }\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n\n final: (callback: (error?: Error) => void) => {\n try {\n flush();\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n }),\n );\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Finished copying ${status.rows} rows into ${tableName} ` +\n `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n );\n return {rows: status.rows, flushTime};\n}\n\nasync function copyText(\n lc: LogContext,\n table: PublishedTableSpec,\n status: DownloadStatus,\n dbClient: PostgresDB,\n from: PostgresTransaction,\n to: Database,\n sampleRate?: number | undefined,\n maxRowsPerTable?: number | undefined,\n) {\n const start = performance.now();\n let flushTime = 0;\n\n const tableName = liteTableName(table);\n const orderedColumns = Object.entries(table.columns);\n\n const columnNames = orderedColumns.map(([c]) => c);\n const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n const valuesSql =\n columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n const insertSql = /*sql*/ `\n INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n const insertStmt = to.prepare(insertSql);\n const insertBatchStmt = to.prepare(\n insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n );\n\n const {select} = makeDownloadStatements(\n table,\n columnNames,\n sampleRate,\n maxRowsPerTable,\n );\n const valuesPerRow = columnSpecs.length;\n const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;\n\n const pendingValues: LiteValueType[] = Array.from({\n length: MAX_BUFFERED_ROWS * valuesPerRow,\n });\n let pendingRows = 0;\n let pendingSize = 0;\n\n function flush() {\n const start = performance.now();\n const flushedRows = pendingRows;\n const flushedSize = pendingSize;\n\n let l = 0;\n for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n }\n for (; pendingRows > 0; pendingRows--) {\n insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n }\n const flushedValues = flushedRows * valuesPerRow;\n for (let i = 0; i < flushedValues; i++) {\n pendingValues[i] = undefined as unknown as LiteValueType;\n }\n pendingSize = 0;\n status.rows += flushedRows;\n\n const elapsed = performance.now() - start;\n flushTime += elapsed;\n lc.debug?.(\n `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n );\n }\n\n lc.info?.(`Starting text copy stream of ${tableName}:`, select);\n const pgParsers = await getTypeParsers(dbClient, {returnJsonAsString: true});\n const parsers = columnSpecs.map(c => {\n const pgParse = pgParsers.getTypeParser(c.typeOID);\n return (val: string) =>\n liteValue(\n pgParse(val) as PostgresValueType,\n c.dataType,\n JSON_STRINGIFIED,\n );\n });\n\n const tsvParser = new TsvParser();\n let col = 0;\n\n await pipeline(\n await from.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n new Writable({\n highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n write(\n chunk: Buffer,\n _encoding: string,\n callback: (error?: Error) => void,\n ) {\n try {\n for (const text of tsvParser.parse(chunk)) {\n pendingSize += text === null ? 4 : text.length;\n pendingValues[pendingRows * valuesPerRow + col] =\n text === null ? null : parsers[col](text);\n\n if (++col === parsers.length) {\n col = 0;\n if (\n ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow ||\n pendingSize >= BUFFERED_SIZE_THRESHOLD\n ) {\n flush();\n }\n }\n }\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n\n final: (callback: (error?: Error) => void) => {\n try {\n flush();\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n }),\n );\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Finished copying ${status.rows} rows into ${tableName} ` +\n `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n );\n return {rows: status.rows, flushTime};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiGA,eAAsB,YACpB,IACA,OACA,IACA,aACA,aACA,SACA;AACA,KAAI,CAAC,0BAA0B,KAAK,MAAM,MAAM,CAC9C,OAAM,IAAI,MACR,2FACD;CAEH,MAAM,EACJ,kBACA,aACA,WAAW,OACX,0BAA0B,OAC1B,WACE;CACJ,MAAM,eAAe,cAAc,MAAM,YAAY,SAAS,GAAG;CACjE,MAAM,MAAM,SAAS,IAAI,aAAa,eAAe;CAIrD,MAAM,qBAAqB,SACvB,KAAA,IACA,SAAS,IAAI,aAAa,oCAAoC;GAC3D,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CAAC;CAEN,MAAM,YAAY,KAAK,KAAK,CAAC,UAAU;CACvC,IAAI;CACJ,MAAM,kBAAkB,2BAA2B,sBACjD,IACA,SAAS,YAAY,KAAK,KAAA,EAC3B,CAAC,QAAQ,IAAI,eAAe;CAC7B,IAAI;AACJ,KAAI;EACF,MAAM,YAAY,MAAM,oBAAoB,IAAI;EAMhD,MAAM,EAAC,iBAAgB,SACnB,MAAM,uBAAuB,KAAK,MAAM,GACxC,MAAM,sBAAsB,IAAI,KAAK,MAAM;AAC/C,KAAG,OAAO,wCAAwC,aAAa,GAAG;EAElE,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,KAAG,OACD,SACI,kCAAkC,SAAS,GAAG,KAAK,kBACnD,kCAAkC,SAAS,GAAG,OACnD;EAED,IAAI;EACJ,IAAI;AAEJ,MAAI,QAAQ;GACV,MAAM,WAAW,MAAM,qCACrB,IACA,YACD;AACD,cAAW,SAAS;AACpB,SAAM,SAAS;AACf,2BAAwB,SAAS;SAC5B;GACL,MAAM,OAAO,MAAM,qBACjB,IACA,KACA,KAAK,mBAAmB,EACxB,OACA,WACA,2BAA2B,aAAA,KAC5B;AACD,cAAW,KAAK;AAChB,SAAM,KAAK;AACX,cAAW,KAAK;;EAGlB,MAAM,iBAAiB,qBAAqB,IAAI;AAEhD,uBAAqB,IAAI,cAAc,gBAAgB,QAAQ;EAG/D,MAAM,QAAQ,YAAY,KAAK;EAE/B,MAAM,YAAY,MAAM,MACtB,KACA,OAAM,OAAM;AACV,SAAM,GAAG,OAAgB,6BAA6B,SAAS,GAAG;AAClE,UAAO,mBAAmB,IAAI,aAAa;KAE7C,EAAC,MAAM,UAAc,CACtB;AAED,uBAAqB,IAAI,UAAU;EAGnC,MAAM,EAAC,QAAQ,YAAW;EAC1B,MAAM,YAAY,OAAO;AACzB,MAAI,UAAU,KAAK,WAAW,mBAAmB,UAC/C,IAAG,OACD,8CAA8C,iBAAiB,MAC1D,UAAU,gDAChB;EAEH,MAAM,aACJ,UAAU,KAAK,UACX,YACA,KAAK,IAAI,kBAAkB,UAAU;EAE3C,MAAM,WAAW,SAAS,IAAI,aAAa,4BAA4B;GACrE,KAAK;IACJ,iBAAiB;GACnB,CAAC;EACF,MAAM,UAAU,sBACd,IACA,UACA,UACA,YACA,UACD;AACD,MAAI;AACF,oBAAiB,IAAI,QAAQ,eAAe;GAC5C,MAAM,aAAa,QAAQ;GAC3B,MAAM,kBAAkB,QAAQ;GAChC,MAAM,YAAY,MAAM,QAAQ,IAC9B,OAAO,KAAI,SACT,QAAQ,iBAAiB,IAAI,OAC3B,wBAAwB,IAAI,IAAI,MAAM,WAAW,KAAA,EAAU,CAC5D,CACF,CACF;AACD,mBAAgB,QACd,IACA,gBACA,WAAW,UAAU,8BAA8B,kBACnD,YACO,EAAC,gBAAgB,UAAU,KAAK,EAAC,aAAY,OAAO,EAAC,EAC7D;AAEI,iBAAc,OAAO;GAC1B,MAAM,YAAY,MAAM,QAAQ,IAC9B,UAAU,KAAI,UACZ,QAAQ,iBAAiB,IAAI,OAC3B,KACE,IACA,OACA,UACA,IACA,IACA,UACA,YACA,gBACD,CACF,CACF,CACF;AACI,iBAAc,eAAe,IAAI,eAAe;AACrD,WAAQ,SAAS;GAEjB,MAAM,QAAQ,UAAU,QACrB,KAAK,UAAU;IACd,MAAM,IAAI,OAAO,KAAK;IACtB,WAAW,IAAI,YAAY,KAAK;IACjC,GACD;IAAC,MAAM;IAAG,WAAW;IAAE,CACxB;AAED,mBAAgB,QACd,IACA,YACA,YAAY,QAAQ,OAAO,WAC3B,IACD;GACD,MAAM,aAAa,YAAY,KAAK;AACpC,qBAAkB,IAAI,QAAQ;GAC9B,MAAM,QAAQ,YAAY,KAAK,GAAG;AAClC,MAAG,OAAO,oBAAoB,MAAM,QAAQ,EAAE,CAAC,MAAM;AAErD,OAAI,YAAY,UACd,OAAM,YAAY,KAAK,OAAO,WAAW,WAAW,QAAQ;QACvD;AACL,WAAO,QAAQ,wDAAwD;IACvE,MAAM,8BAAc,IAAI,KAAqB;AAC7C,SAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,IACpC,aAAY,IAAI,UAAU,GAAG,OAAO,OAAO,UAAU,GAAG,KAAK;AAE/D,wBAAoB,IAAI,IAAI,WAAW,YAAY;;GAGrD,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,MAAG,OACD,UAAU,MAAM,KAAK,gBAAgB,CAAC,WAAW,UAAU,aAAa,aAAa,SAAS,IAAI,WACrF,MAAM,UAAU,QAAQ,EAAE,CAAC,WAAW,MAAM,QAAQ,EAAE,CAAC,WAAW,QAAQ,QAAQ,EAAE,CAAC,MACnG;YACO;AAEH,YAAS,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,0BAA0B,EAAE,CAAC;;UAEjE,GAAG;AACV,MAAI,UAAU;AAIZ,MAAG,OAAO,6BAA6B,YAAY,EAAE;AACrD,SAAM,GAAG;;8BAEe,SAAS;QAC/B,OAAM,MAAK,GAAG,OAAO,mCAAmC,YAAY,EAAE,CAAC;;AAE3E,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;AACtB,MAAI,sBACF,OAAM,uBAAuB,CAAC,OAAM,MAClC,GAAG,OAAO,mCAAmC,EAAE,CAChD;AAEH,MAAI,mBACF,OAAM,mBAAmB,KAAK;AAEhC,QAAM,IAAI,KAAK;;;;;;;;;;;;;AAwBnB,eAAsB,kBACpB,IACA,OACA,aACA,QACA,SACA,aACe;CACf,MAAM,MAAM,MAAM,QAChB,KAAK,OAAO,aAAa,QAAQ,EAAE,oBAAoB,CACxD;CAED,MAAM,KAAK,IAAI,SAAS,IADT,KAAK,KAAK,oBAAoB,CACV;AACnC,KAAI;AACF,QAAM,YACJ,IACA,OACA,IACA,aACA;GAGE,kBAAkB;GAClB,UAAU,aAAa;GACvB;GACD,EACD,QACD;WACO;AACR,MAAI;AACF,MAAG,OAAO;WACH,GAAG;AACV,MAAG,OAAO,mCAAmC,EAAE;;AAEjD,QAAM,GAAG,KAAK;GAAC,WAAW;GAAM,OAAO;GAAK,CAAC,CAAC,OAAM,MAClD,GAAG,OAAO,wCAAwC,OAAO,EAAE,CAC5D;;;AAIL,eAAe,oBAAoB,KAAiB;CAClD,MAAM,EAAC,UAAU,aACf,MAAM,GAA0C;;;KAIhD;AAEF,KAAI,aAAa,UACf,OAAM,IAAI,MACR,uEAAuE,SAAS,GACjF;AAEH,KAAI,UAAA,KACF,OAAM,IAAI,MACR,sDAAsD,QAAQ,IAC/D;AAEH,QAAO;;AAGT,eAAe,sBACb,IACA,KACA,OACA,WAAW,MACwB;CACnC,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,IAAG,OAAO,oCAAoC,SAAS,GAAG,OAAO;AAEjE,OAAM,kBAAkB,IAAI,KAAK,MAAM;CACvC,MAAM,EAAC,iBAAgB,MAAM,uBAAuB,KAAK,MAAM;AAE/D,KAAI,UAAU;EACZ,IAAI,QAAQ;EACZ,MAAM,0BAA0B,aAAa,QAC3C,MAAK,CAAC,EAAE,WAAW,IAAI,CACxB;EACD,MAAM,SAAS,MAAM,GAAG;4DACgC,IAAI,aAAa,CAAC;QACtE,QAAQ;AACZ,MAAI,OAAO,WAAW,aAAa,OACjC,IAAG,OACD,iCAAiC,aAAa,kBACxC,OAAO,MAAM,CAAC,cACrB;WAED,CAAC,OAAO,IAAI,IAAI,MAAM,aAAa,EAAE,IAAI,IAAI,wBAAwB,CAAC,CAEtE,IAAG,OACD,2BAA2B,MAAM,aAAa,sCAC3B,wBAAwB,cAC5C;MAED,SAAQ;AAEV,MAAI,CAAC,OAAO;AACV,SAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,SAAS,CAAC;AACxD,UAAO,sBAAsB,IAAI,KAAK,OAAO,MAAM;;;AAGvD,QAAO,EAAC,cAAa;;AAGvB,SAAS,sBACP,IACA,IACA,UACA,YACA,WACiB;CACjB,MAAM,EAAC,SAAQ,eAAe,SAAS;CACvC,MAAM,eAAe,IAAI,gBAAgB,IAAI;EAC3C,MAAM;EACN;EACA,gBAAgB;EACjB,CAAC;AACF,cAAa,IAAI,GAAG;AAEpB,IAAG,OAAO,WAAW,WAAW,mBAAmB,UAAU,SAAS;AAEtE,KAAI,SAAS,QAAQ,SAAS,KAAK,GAAG,GACpC,IAAG,OACD,mUAMD;AAEH,QAAO;;;;;;;;;;;;AAaT,eAAe,qCACb,IACA,aAKC;CACD,MAAM,SAAS,SAAS,IAAI,aAAa,gCAAgC,EACvE,KAAK,GACN,CAAC;CACF,MAAM,QAAQ,UAA2C;CACzD,MAAM,UAAU,UAAgB;CAChC,MAAM,OAAO,OACV,MAAM,UAAe,OAAM,OAAM;AAChC,QAAM,EAAE,oDAAoD,SAAS;EACrE,MAAM,CAAC,OAAO,MAAM,EAAqC;;;AAGzD,QAAM,QAAQ,IAAI;AAClB,QAAM,QAAQ;GACd,CACD,OAAM,MAAK,MAAM,OAAO,EAAE,CAAC;CAE9B,IAAI;CACJ,IAAI;AACJ,KAAI;AACF,GAAC,CAAC,UAAU,OAAO,MAAM,MAAM;UACxB,GAAG;AACV,QAAM,OACH,KAAK,CACL,OAAM,QACL,GAAG,OAAO,qDAAqD,IAAI,CACpE;AACH,QAAM;;AAER,IAAG,OACD,qBAAqB,SAAS,UAAU,IAAI,wBAC7C;AACD,QAAO;EACL;EACA;EACA,SAAS,YAAY;AACnB,WAAQ,SAAS;AACjB,OAAI;AACF,UAAM;YACC,GAAG;AACV,OAAG,OAAO,gDAAgD,EAAE;;AAE9D,SAAM,OAAO,KAAK;;EAErB;;AAGH,SAAS,iBACP,IACA,QACA,gBACA;CAGA,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,GAAG,CAAC;AAChE,MAAK,MAAM,KAAK,QAAQ;AACtB,KAAG,KAAK,yBAAyB,kBAAkB,GAAG,eAAe,CAAC,CAAC;EACvE,MAAM,YAAY,cAAc,EAAE;AAClC,OAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,EAAE,QAAQ,CACxD,gBAAe,OAAO,WAAW,SAAS,QAAQ;;;AAKxD,SAAS,kBAAkB,IAAc,SAAsB;AAC7D,MAAK,MAAM,SAAS,QAClB,IAAG,KAAK,yBAAyB,uBAAuB,MAAM,CAAC,CAAC;;;;;;;;;;AAYpE,SAAgB,oBACd,IACA,IACA,WACA,aACM;CACN,MAAM,SAAmB,EAAE;CAC3B,IAAI,iBAAiB;CACrB,IAAI,cAAc;CAIlB,MAAM,aAAa,WAAW,GAAG;CACjC,MAAM,kBAAkB,IAAI,IAAI,WAAW,KAAI,MAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;AACjE,MAAK,MAAM,MAAM,UAAU,QAAQ;EACjC,MAAM,OAAO,cAAc,GAAG;EAC9B,MAAM,OAAO,gBAAgB,IAAI,KAAK;AACtC,MAAI,CAAC,MAAM;AACT,UAAO,KAAK,6BAA6B,OAAO;AAChD;;AAEF,OAAK,MAAM,OAAO,OAAO,KAAK,GAAG,QAAQ,EAAE;AACzC;AACA,OAAI,EAAE,OAAO,KAAK,SAChB,QAAO,KAAK,mCAAmC,KAAK,IAAI,MAAM;;;CAMpE,MAAM,iBAAiB,IAAI,IAAI,YAAY,GAAG,CAAC,KAAI,MAAK,EAAE,KAAK,CAAC;AAChE,MAAK,MAAM,MAAM,UAAU,SAAS;EAClC,MAAM,SAAS,uBAAuB,GAAG;AACzC,MAAI,CAAC,eAAe,IAAI,OAAO,KAAK,CAClC,QAAO,KACL,6BAA6B,OAAO,KAAK,MAAM,OAAO,YACvD;;AAKL,MAAK,MAAM,CAAC,OAAO,aAAa,YAC9B,KAAI;EACF,MAAM,CAAC,OAAO,GACX,QAAQ,kCAAkC,MAAM,GAAG,CACnD,KAAsB;AACzB,MAAI,IAAI,UAAU,SAChB,QAAO,KACL,gCAAgC,MAAM,0BACX,SAAS,gBAAgB,IAAI,QACzD;MAED,gBAAe,IAAI;UAEd,GAAG;AACV,SAAO,KAAK,iCAAiC,MAAM,IAAI,OAAO,EAAE,GAAG;;CAMvE,MAAM,aAAa,gBAAgB,IAAI,IAAI,EACzC,2BAA2B,OAC5B,CAAC;AACF,MAAK,MAAM,MAAM,UAAU,QAAQ;EACjC,MAAM,OAAO,cAAc,GAAG;AAC9B,MAAI,CAAC,WAAW,IAAI,KAAK,CACvB,QAAO,KACL,6DAA6D,OAC9D;;CAKL,MAAM,OAAO,KAAK,oBAAoB,YAAY,GAAG,CAAC;AACtD,MAAK,MAAM,MAAM,UAAU,QAAQ;EACjC,MAAM,OAAO,cAAc,GAAG;EAC9B,MAAM,OAAO,KAAK,SAAS,KAAK;AAChC,OAAK,MAAM,OAAO,OAAO,KAAK,GAAG,QAAQ,CACvC,KAAI,CAAC,KAAK,IAAI,IAAI,CAChB,QAAO,KAAK,mCAAmC,KAAK,GAAG,MAAM;;AAKnE,KAAI,OAAO,OACT,OAAM,IAAI,MACR,uCAAuC,OAAO,OAAO,iBACnD,OAAO,KAAI,MAAK,OAAO,IAAI,CAAC,KAAK,KAAK,CACzC;AAGH,IAAG,OACD,uCACK,UAAU,OAAO,OAAO,WACxB,UAAU,QAAQ,OAAO,YACzB,eAAe,YACf,YAAY,gBAAgB,CAAC,OACnC;;AASH,IAAM,KAAK,OAAO;AAClB,IAAM,oBAAoB;AAC1B,IAAM,0BAA0B,IAAI;;;;;;;AAcpC,SAAS,kBAAkB,YAAwC;AACjE,KAAI,eAAe,KAAA,KAAa,cAAc,EAC5C,QAAO;AAKT,QAAe,0BADH,YAAY,aAAa,KAAK,QAAQ,EAAE,CAAC,CACR;;AAG/C,SAAS,YAAY,iBAA6C;AAChE,QAAO,oBAAoB,KAAA,IACf,UAAU,oBAClB;;;;;;AAON,SAAgB,sBACd,OACA,MACU;AACV,QAAO,KAAK,KAAI,QAAO;EACrB,MAAM,OAAO,MAAM,QAAQ;AAC3B,SAAO,iBAAiB,KAAK,GAAG,GAAG,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC;GACrD;;AAGJ,SAAgB,uBACd,OACA,MACA,YACA,iBACA,aACoB;CACpB,MAAM,mBAAmB,OAAO,OAAO,MAAM,aAAa,CACvD,KAAK,EAAC,gBAAe,UAAU,CAC/B,QAAO,MAAK,CAAC,CAAC,EAAE;CACnB,MAAM,QACJ,iBAAiB,WAAW,IACxB,KACQ,SAAS,iBAAiB,KAAK,OAAO;CACpD,MAAM,SAAS,kBAAkB,WAAW;CAC5C,MAAM,QAAQ,YAAY,gBAAgB;CAC1C,MAAM,YAAoB,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,GAAG,OAAO,GAAG;CACjF,MAAM,SAAiB,WAAW,eAAe,KAAK,IAAI,GAAG,EAAE,KAAK,IAAI,CAAC,GAAG,YAAY;AACxF,KAAI,OAAO;EAGT,MAAM,YAAY,KACf,KAAI,QAAO,2BAA2B,GAAG,IAAI,CAAC,OAAO,CACrD,KAAK,MAAM;AACd,SAAO;GACL;GACA,cAAsB,8DAA8D,YAAY,MAAM;GACtG,eAAuB,oEAAoE,UAAU,SAAS,YAAY,MAAM;GACjI;;CAEH,MAAM,aAAa,IAAI,KAAK,KAAI,QAAO,+BAA+B,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;AACnG,QAAO;EACL;EACA,cAAsB,kCAAkC;EACxD,eAAuB,UAAU,WAAW,mBAAmB;EAChE;;AASH,eAAsB,wBACpB,IACA,KACA,MACA,YACwB;CACxB,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,QAAQ,cAAc,KAAK;CACjC,MAAM,UAAU,OAAO,KAAK,KAAK,QAAQ;AACzC,KAAI,WAGF,QAAO;EACL;EACA,QAAQ;GAAC;GAAO;GAAS,MAAM;GAAG,WAAW;GAAG,YAAY;GAAE;EAC/D;CAaH,MAAM,EAAC,WAAW,gBAPK,MAAM,GAE5B;;;kBAHqB,GAAG,GAAG,KAAK,OAAO,CAAC,GAAG,GAAG,KAAK,KAAK,GAM3B,aAEiB,MAAM;EACnD,WAAW;EACX,YAAY;EACb;CAED,MAAM,QAAuB;EAC3B;EACA,QAAQ;GACN;GACA;GACA,MAAM;GACN;GACA;GACD;EACF;CACD,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,IAAG,OAAO,uCAAuC,MAAM,IAAI,QAAQ,OAAO,EACxE,OAAO,MAAM,QACd,CAAC;AACF,QAAO;;AAGT,SAAS,KACP,IACA,EAAC,MAAM,OAAO,UACd,UACA,MACA,IACA,UACA,YACA,iBACA;AACA,KAAI,SACF,QAAO,SACL,IACA,OACA,QACA,UACA,MACA,IACA,YACA,gBACD;AAEH,QAAO,WAAW,IAAI,OAAO,QAAQ,MAAM,IAAI,YAAY,gBAAgB;;AAG7E,eAAe,WACb,IACA,OACA,QACA,MACA,IACA,YACA,iBACA;CACA,MAAM,QAAQ,YAAY,KAAK;CAC/B,IAAI,YAAY;CAEhB,MAAM,YAAY,cAAc,MAAM;CACtC,MAAM,iBAAiB,OAAO,QAAQ,MAAM,QAAQ;CAEpD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,EAAE;CAClD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,UAAU,KAAK;CAC/D,MAAM,mBAAmB,YAAY,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI;CAE9D,MAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,EAAE,CAAC,MAAM;CACzE,MAAM,YAAoB;mBACT,UAAU,KAAK,iBAAiB,WAAW;CAC5D,MAAM,aAAa,GAAG,QAAQ,UAAU;CACxC,MAAM,kBAAkB,GAAG,QACzB,YAAY,IAAI,YAAY,OAAA,GAA6B,CAC1D;CAGD,MAAM,SAAS,uBACb,OACA,aACA,YACA,iBACA,sBAAsB,OAAO,YAAY,CAC1C,CAAC;CAEF,MAAM,WAAW,eAAe,KAAK,GAAG,UACtC,iBAAiB,KAAK,GAAG,kBAAkB,KAAK,GAAG,gBACpD;CAED,MAAM,eAAe,YAAY;CACjC,MAAM,iBAAiB,eAAA;CAEvB,MAAM,gBAAiC,MAAM,KAAK,EAChD,QAAQ,oBAAoB,cAC7B,CAAC;CACF,IAAI,cAAc;CAClB,IAAI,cAAc;CAElB,SAAS,QAAQ;EACf,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,cAAc;EACpB,MAAM,cAAc;EAEpB,IAAI,IAAI;AACR,SAAO,cAAA,IAAiC,eAAA,GACtC,iBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,eAAgB,CAAC;AAEpE,SAAO,cAAc,GAAG,cACtB,YAAW,IAAI,cAAc,MAAM,GAAI,KAAK,aAAc,CAAC;EAE7D,MAAM,gBAAgB,cAAc;AACpC,OAAK,IAAI,IAAI,GAAG,IAAI,eAAe,IACjC,eAAc,KAAK,KAAA;AAErB,gBAAc;AACd,SAAO,QAAQ;EAEf,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,eAAa;AACb,KAAG,QACD,WAAW,YAAY,GAAG,UAAU,SAAS,YAAY,aAAa,QAAQ,QAAQ,EAAE,CAAC,KAC1F;;CAGH,MAAM,eAAe,IAAI,kBAAkB;CAC3C,IAAI,MAAM;AAEV,IAAG,OAAO,kCAAkC,UAAU,IAAI,OAAO;AAEjE,OAAM,WACJ,MAAM,KACH,OAAO,SAAS,OAAO,kCAAkC,CACzD,UAAU,EACb,IAAI,SAAS;EACX,eAAe;EAEf,MACE,OACA,WACA,UACA;AACA,OAAI;AACF,SAAK,MAAM,YAAY,aAAa,MAAM,MAAM,EAAE;AAChD,oBAAe,aAAa,OAAO,IAAI,SAAS;AAChD,mBAAc,cAAc,eAAe,OACzC,aAAa,OAAO,OAAO,SAAS,KAAK,SAAS;AAEpD,SAAI,EAAE,QAAQ,SAAS,QAAQ;AAC7B,YAAM;AACN,UACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,wBAEf,QAAO;;;AAIb,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAI3D,QAAQ,aAAsC;AAC5C,OAAI;AACF,WAAO;AACP,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAG5D,CAAC,CACH;CAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,IAAG,OACD,oBAAoB,OAAO,KAAK,aAAa,UAAU,WAC1C,UAAU,QAAQ,EAAE,CAAC,eAAe,QAAQ,QAAQ,EAAE,CAAC,OACrE;AACD,QAAO;EAAC,MAAM,OAAO;EAAM;EAAU;;AAGvC,eAAe,SACb,IACA,OACA,QACA,UACA,MACA,IACA,YACA,iBACA;CACA,MAAM,QAAQ,YAAY,KAAK;CAC/B,IAAI,YAAY;CAEhB,MAAM,YAAY,cAAc,MAAM;CACtC,MAAM,iBAAiB,OAAO,QAAQ,MAAM,QAAQ;CAEpD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,EAAE;CAClD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,UAAU,KAAK;CAC/D,MAAM,mBAAmB,YAAY,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI;CAE9D,MAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,EAAE,CAAC,MAAM;CACzE,MAAM,YAAoB;mBACT,UAAU,KAAK,iBAAiB,WAAW;CAC5D,MAAM,aAAa,GAAG,QAAQ,UAAU;CACxC,MAAM,kBAAkB,GAAG,QACzB,YAAY,IAAI,YAAY,OAAA,GAA6B,CAC1D;CAED,MAAM,EAAC,WAAU,uBACf,OACA,aACA,YACA,gBACD;CACD,MAAM,eAAe,YAAY;CACjC,MAAM,iBAAiB,eAAA;CAEvB,MAAM,gBAAiC,MAAM,KAAK,EAChD,QAAQ,oBAAoB,cAC7B,CAAC;CACF,IAAI,cAAc;CAClB,IAAI,cAAc;CAElB,SAAS,QAAQ;EACf,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,cAAc;EACpB,MAAM,cAAc;EAEpB,IAAI,IAAI;AACR,SAAO,cAAA,IAAiC,eAAA,GACtC,iBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,eAAgB,CAAC;AAEpE,SAAO,cAAc,GAAG,cACtB,YAAW,IAAI,cAAc,MAAM,GAAI,KAAK,aAAc,CAAC;EAE7D,MAAM,gBAAgB,cAAc;AACpC,OAAK,IAAI,IAAI,GAAG,IAAI,eAAe,IACjC,eAAc,KAAK,KAAA;AAErB,gBAAc;AACd,SAAO,QAAQ;EAEf,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,eAAa;AACb,KAAG,QACD,WAAW,YAAY,GAAG,UAAU,SAAS,YAAY,aAAa,QAAQ,QAAQ,EAAE,CAAC,KAC1F;;AAGH,IAAG,OAAO,gCAAgC,UAAU,IAAI,OAAO;CAC/D,MAAM,YAAY,MAAM,eAAe,UAAU,EAAC,oBAAoB,MAAK,CAAC;CAC5E,MAAM,UAAU,YAAY,KAAI,MAAK;EACnC,MAAM,UAAU,UAAU,cAAc,EAAE,QAAQ;AAClD,UAAQ,QACN,UACE,QAAQ,IAAI,EACZ,EAAE,UAAA,IAEH;GACH;CAEF,MAAM,YAAY,IAAI,WAAW;CACjC,IAAI,MAAM;AAEV,OAAM,WACJ,MAAM,KAAK,OAAO,SAAS,OAAO,aAAa,CAAC,UAAU,EAC1D,IAAI,SAAS;EACX,eAAe;EAEf,MACE,OACA,WACA,UACA;AACA,OAAI;AACF,SAAK,MAAM,QAAQ,UAAU,MAAM,MAAM,EAAE;AACzC,oBAAe,SAAS,OAAO,IAAI,KAAK;AACxC,mBAAc,cAAc,eAAe,OACzC,SAAS,OAAO,OAAO,QAAQ,KAAK,KAAK;AAE3C,SAAI,EAAE,QAAQ,QAAQ,QAAQ;AAC5B,YAAM;AACN,UACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,wBAEf,QAAO;;;AAIb,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAI3D,QAAQ,aAAsC;AAC5C,OAAI;AACF,WAAO;AACP,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAG5D,CAAC,CACH;CAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,IAAG,OACD,oBAAoB,OAAO,KAAK,aAAa,UAAU,WAC1C,UAAU,QAAQ,EAAE,CAAC,eAAe,QAAQ,QAAQ,EAAE,CAAC,OACrE;AACD,QAAO;EAAC,MAAM,OAAO;EAAM;EAAU"}

package/out/zero-cache/src/services/change-source/pg/logical-replication/binary-reader.js.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"binary-reader.js","names":["#b","#p"],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/binary-reader.ts"],"sourcesContent":["// Forked from https://github.com/kibae/pg-logical-replication/blob/c55abddc62eadd61bd38922037ecb7a1469fa8c3/src/output-plugins/pgoutput/binary-reader.ts\n\n// should not use { fatal: true } because ErrorResponse can use invalid utf8 chars\nconst textDecoder = new TextDecoder();\n\n// https://www.postgresql.org/docs/14/protocol-message-types.html\nexport class BinaryReader {\n #p = 0;\n readonly #b: Uint8Array;\n\n constructor(b: Uint8Array) {\n this.#b = b;\n }\n\n readUint8() {\n this.checkSize(1);\n\n return this.#b[this.#p++];\n }\n\n readInt16() {\n this.checkSize(2);\n\n return (this.#b[this.#p++] << 8) | this.#b[this.#p++];\n }\n\n readInt32() {\n this.checkSize(4);\n\n return (\n (this.#b[this.#p++] << 24) |\n (this.#b[this.#p++] << 16) |\n (this.#b[this.#p++] << 8) |\n this.#b[this.#p++]\n );\n }\n\n readString() {\n const endIdx = this.#b.indexOf(0x00, this.#p);\n\n if (endIdx < 0) {\n // TODO PgError.protocol_violation\n throw Error('unexpected end of message');\n }\n\n const strBuf = this.#b.subarray(this.#p, endIdx);\n this.#p = endIdx + 1;\n\n return this.decodeText(strBuf);\n }\n\n decodeText(strBuf: Uint8Array) {\n return textDecoder.decode(strBuf);\n }\n\n read(n: number) {\n this.checkSize(n);\n\n return this.#b.subarray(this.#p, (this.#p += n));\n }\n\n checkSize(n: number) {\n if (this.#b.length < this.#p + n) {\n // TODO PgError.protocol_violation\n throw Error('unexpected end of message');\n }\n }\n\n array<T>(length: number, fn: () => T): T[] {\n return Array.from({length}, fn, this);\n }\n\n // replication helpers\n readLsn() {\n const h = this.readUint32();\n const l = this.readUint32();\n\n if (h === 0 && l === 0) {\n return null;\n }\n\n return `${h.toString(16).padStart(8, '0')}/${l\n .toString(16)\n .padStart(8, '0')}`.toUpperCase();\n }\n\n readTime() {\n // (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * USECS_PER_DAY == 946684800000000\n return this.readUint64() + BigInt('946684800000000');\n }\n\n readUint64() {\n return (\n (BigInt(this.readUint32()) << BigInt(32)) | BigInt(this.readUint32())\n );\n }\n\n readUint32() {\n return this.readInt32() >>> 0;\n }\n}\n"],"mappings":";AAGA,IAAM,cAAc,IAAI,aAAa;AAGrC,IAAa,eAAb,MAA0B;CACxB,KAAK;CACL;CAEA,YAAY,GAAe;AACzB,QAAA,IAAU;;CAGZ,YAAY;AACV,OAAK,UAAU,EAAE;AAEjB,SAAO,MAAA,EAAQ,MAAA;;CAGjB,YAAY;AACV,OAAK,UAAU,EAAE;AAEjB,SAAQ,MAAA,EAAQ,MAAA,QAAc,IAAK,MAAA,EAAQ,MAAA;;CAG7C,YAAY;AACV,OAAK,UAAU,EAAE;AAEjB,SACG,MAAA,EAAQ,MAAA,QAAc,KACtB,MAAA,EAAQ,MAAA,QAAc,KACtB,MAAA,EAAQ,MAAA,QAAc,IACvB,MAAA,EAAQ,MAAA;;CAIZ,aAAa;EACX,MAAM,SAAS,MAAA,EAAQ,QAAQ,GAAM,MAAA,EAAQ;AAE7C,MAAI,SAAS,EAEX,OAAM,MAAM,4BAA4B;EAG1C,MAAM,SAAS,MAAA,EAAQ,SAAS,MAAA,GAAS,OAAO;AAChD,QAAA,IAAU,SAAS;AAEnB,SAAO,KAAK,WAAW,OAAO;;CAGhC,WAAW,QAAoB;AAC7B,SAAO,YAAY,OAAO,OAAO;;CAGnC,KAAK,GAAW;AACd,OAAK,UAAU,EAAE;AAEjB,SAAO,MAAA,EAAQ,SAAS,MAAA,GAAU,MAAA,KAAW,EAAG;;CAGlD,UAAU,GAAW;AACnB,MAAI,MAAA,EAAQ,SAAS,MAAA,IAAU,EAE7B,OAAM,MAAM,4BAA4B;;CAI5C,MAAS,QAAgB,IAAkB;AACzC,SAAO,MAAM,KAAK,EAAC,QAAO,EAAE,IAAI,KAAK;;CAIvC,UAAU;EACR,MAAM,IAAI,KAAK,YAAY;EAC3B,MAAM,IAAI,KAAK,YAAY;AAE3B,MAAI,MAAM,KAAK,MAAM,EACnB,QAAO;AAGT,SAAO,GAAG,EAAE,SAAS,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAC1C,SAAS,GAAG,CACZ,SAAS,GAAG,IAAI,GAAG,aAAa;;CAGrC,WAAW;AAET,SAAO,KAAK,YAAY,GAAG,OAAO,kBAAkB;;CAGtD,aAAa;AACX,SACG,OAAO,KAAK,YAAY,CAAC,IAAI,OAAO,GAAG,GAAI,OAAO,KAAK,YAAY,CAAC;;CAIzE,aAAa;AACX,SAAO,KAAK,WAAW,KAAK"}

package/out/zero-cache/src/services/change-source/pg/logical-replication/pgoutput-parser.js.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"pgoutput-parser.js","names":["#typeParsers","#getTypeParser"],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/pgoutput-parser.ts"],"sourcesContent":["// Forked from https://github.com/kibae/pg-logical-replication/blob/c55abddc62eadd61bd38922037ecb7a1469fa8c3/src/output-plugins/pgoutput/pgoutput-parser.ts\n/* oxlint-disable */\n\n// https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html\n\n// Replaced by TypeParsers\n// import {types} from 'pg';\n\nexport type TypeParser = (val: string) => unknown;\n\nimport type {TypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {BinaryReader} from './binary-reader.ts';\nimport type {\n Message,\n MessageBegin,\n MessageCommit,\n MessageDelete,\n MessageInsert,\n MessageMessage,\n MessageOrigin,\n MessageRelation,\n MessageTruncate,\n MessageType,\n MessageUpdate,\n RelationColumn,\n} from './pgoutput.types.ts';\n\nexport class PgoutputParser {\n _typeCache = new Map<number, {typeSchema: string; typeName: string}>();\n _relationCache = new Map<number, MessageRelation>();\n\n // Replaces \"pg-types\" library.\n #typeParsers: TypeParsers;\n constructor(typeParsers: TypeParsers) {\n this.#typeParsers = typeParsers;\n }\n #getTypeParser(typeOid: number): TypeParser {\n return this.#typeParsers.getTypeParser(typeOid);\n }\n // Replaced \"pg-types\" library.\n\n public parse(buf: Buffer): Message {\n const reader = new BinaryReader(buf);\n const tag = reader.readUint8();\n\n switch (tag) {\n case 0x42 /*B*/:\n return this.msgBegin(reader);\n case 0x4f /*O*/:\n return this.msgOrigin(reader);\n case 0x59 /*Y*/:\n return this.msgType(reader);\n case 0x52 /*R*/:\n return this.msgRelation(reader);\n case 0x49 /*I*/:\n return this.msgInsert(reader);\n case 0x55 /*U*/:\n return this.msgUpdate(reader);\n case 0x44 /*D*/:\n return this.msgDelete(reader);\n case 0x54 /*T*/:\n return this.msgTruncate(reader);\n case 0x4d /*M*/:\n return this.msgMessage(reader);\n case 0x43 /*C*/:\n return this.msgCommit(reader);\n default:\n throw Error('unknown pgoutput message');\n }\n }\n\n private msgBegin(reader: BinaryReader): MessageBegin {\n // TODO lsn can be null if origin sended\n // https://github.com/postgres/postgres/blob/85c61ba8920ba73500e1518c63795982ee455d14/src/backend/replication/pgoutput/pgoutput.c#L409\n // https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/include/replication/reorderbuffer.h#L275\n\n return {\n tag: 'begin',\n commitLsn: reader.readLsn(),\n commitTime: reader.readTime(),\n xid: reader.readInt32(),\n };\n }\n\n private msgOrigin(reader: BinaryReader): MessageOrigin {\n return {\n tag: 'origin',\n originLsn: reader.readLsn(),\n originName: reader.readString(),\n };\n }\n\n private msgType(reader: BinaryReader): MessageType {\n const typeOid = reader.readInt32();\n const typeSchema = reader.readString();\n const typeName = reader.readString();\n\n // mem leak not likely to happen because amount of types is usually small\n this._typeCache.set(typeOid, {typeSchema, typeName});\n\n return {tag: 'type', typeOid, typeSchema, typeName};\n }\n\n private msgRelation(reader: BinaryReader): MessageRelation {\n // lsn expected to be null\n // https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/backend/replication/walsender.c#L1342\n const relationOid = reader.readInt32();\n const schema = reader.readString();\n const name = reader.readString();\n const replicaIdentity = this.readRelationReplicaIdentity(reader);\n const columns = reader.array(reader.readInt16(), () =>\n this.readRelationColumn(reader),\n );\n const keyColumns = columns.filter(it => it.flags & 0b1).map(it => it.name);\n\n const msg: MessageRelation = {\n tag: 'relation',\n relationOid,\n schema,\n name,\n replicaIdentity,\n columns,\n keyColumns,\n };\n\n // mem leak not likely to happen because amount of relations is usually small\n this._relationCache.set(relationOid, msg);\n\n return msg;\n }\n\n private readRelationReplicaIdentity(reader: BinaryReader) {\n // https://www.postgresql.org/docs/14/catalog-pg-class.html\n const ident = reader.readUint8();\n\n switch (ident) {\n case 0x64 /*d*/:\n return 'default';\n case 0x6e /*n*/:\n return 'nothing';\n case 0x66 /*f*/:\n return 'full';\n case 0x69 /*i*/:\n return 'index';\n default:\n throw Error(`unknown replica identity ${String.fromCharCode(ident)}`);\n }\n }\n\n private readRelationColumn(reader: BinaryReader): RelationColumn {\n const flags = reader.readUint8();\n const name = reader.readString();\n const typeOid = reader.readInt32();\n const typeMod = reader.readInt32();\n\n return {\n flags,\n name,\n typeOid,\n typeMod,\n typeSchema: null,\n typeName: null, // TODO resolve builtin type names?\n ...this._typeCache.get(typeOid),\n // parser: types.getTypeParser(typeOid),\n parser: this.#getTypeParser(typeOid),\n };\n }\n\n private msgInsert(reader: BinaryReader): MessageInsert {\n const relation = this._relationCache.get(reader.readInt32());\n\n if (!relation) {\n throw Error('missing relation');\n }\n\n reader.readUint8(); // consume the 'N' key\n\n return {\n tag: 'insert',\n relation,\n new: this.readTuple(reader, relation),\n };\n }\n\n private msgUpdate(reader: BinaryReader): MessageUpdate {\n const relation = this._relationCache.get(reader.readInt32());\n\n if (!relation) {\n throw Error('missing relation');\n }\n\n let key: Record<string, any> | null = null;\n let old: Record<string, any> | null = null;\n let new_: Record<string, any> | null = null;\n const subMsgKey = reader.readUint8();\n\n if (subMsgKey === 0x4b /*K*/) {\n key = this.readKeyTuple(reader, relation);\n reader.readUint8(); // consume the 'N' key\n new_ = this.readTuple(reader, relation);\n } else if (subMsgKey === 0x4f /*O*/) {\n old = this.readTuple(reader, relation);\n reader.readUint8(); // consume the 'N' key\n new_ = this.readTuple(reader, relation, old);\n } else if (subMsgKey === 0x4e /*N*/) {\n new_ = this.readTuple(reader, relation);\n } else {\n throw Error(`unknown submessage key ${String.fromCharCode(subMsgKey)}`);\n }\n\n return {tag: 'update', relation, key, old, new: new_};\n }\n\n private msgDelete(reader: BinaryReader): MessageDelete {\n const relation = this._relationCache.get(reader.readInt32());\n\n if (!relation) {\n throw Error('missing relation');\n }\n\n let key: Record<string, any> | null = null;\n let old: Record<string, any> | null = null;\n const subMsgKey = reader.readUint8();\n\n if (subMsgKey === 0x4b /*K*/) {\n key = this.readKeyTuple(reader, relation);\n } else if (subMsgKey === 0x4f /*O*/) {\n old = this.readTuple(reader, relation);\n } else {\n throw Error(`unknown submessage key ${String.fromCharCode(subMsgKey)}`);\n }\n\n return {tag: 'delete', relation, key, old};\n }\n\n private readKeyTuple(\n reader: BinaryReader,\n relation: MessageRelation,\n ): Record<string, any> {\n const tuple = this.readTuple(reader, relation);\n const key = Object.create(null);\n\n for (const k of relation.keyColumns) {\n // If value is `null`, then it is definitely not part of key,\n // because key cannot have nulls by documentation.\n // And if we got `null` while reading keyOnly tuple,\n // then it means that `null` is not actual value\n // but placeholder of non-key column.\n key[k] = tuple[k] === null ? undefined : tuple[k];\n }\n\n return key;\n }\n\n private readTuple(\n reader: BinaryReader,\n {columns}: MessageRelation,\n unchangedToastFallback?: Record<string, any> | null,\n ): Record<string, any> {\n const nfields = reader.readInt16();\n const tuple = Object.create(null);\n\n for (let i = 0; i < nfields; i++) {\n const {name, parser} = columns[i];\n const kind = reader.readUint8();\n\n switch (kind) {\n case 0x62: // 'b' binary\n const bsize = reader.readInt32();\n const bval = reader.read(bsize);\n // dont need to .slice() because new buffer\n // is created for each replication chunk\n tuple[name] = bval;\n break;\n case 0x74: // 't' text\n const valsize = reader.readInt32();\n const valbuf = reader.read(valsize);\n const valtext = reader.decodeText(valbuf);\n tuple[name] = parser(valtext);\n break;\n case 0x6e: // 'n' null\n tuple[name] = null;\n break;\n case 0x75: // 'u' unchanged toast datum\n tuple[name] = unchangedToastFallback?.[name];\n break;\n default:\n throw Error(`unknown attribute kind ${String.fromCharCode(kind)}`);\n }\n }\n\n return tuple;\n }\n\n private msgTruncate(reader: BinaryReader): MessageTruncate {\n const nrels = reader.readInt32();\n const flags = reader.readUint8();\n\n return {\n tag: 'truncate',\n cascade: Boolean(flags & 0b1),\n restartIdentity: Boolean(flags & 0b10),\n relations: reader.array(\n nrels,\n () => this._relationCache.get(reader.readInt32()) as MessageRelation,\n ),\n };\n }\n\n private msgMessage(reader: BinaryReader): MessageMessage {\n const flags = reader.readUint8();\n\n return {\n tag: 'message',\n flags,\n transactional: Boolean(flags & 0b1),\n messageLsn: reader.readLsn(),\n prefix: reader.readString(),\n content: reader.read(reader.readInt32()),\n };\n }\n\n private msgCommit(reader: BinaryReader): MessageCommit {\n return {\n tag: 'commit',\n flags: reader.readUint8(), // reserved unused\n commitLsn: reader.readLsn(), // should be the same as begin.commitLsn\n commitEndLsn: reader.readLsn(),\n commitTime: reader.readTime(),\n };\n }\n}\n"],"mappings":";;AA2BA,IAAa,iBAAb,MAA4B;CAC1B,6BAAa,IAAI,KAAqD;CACtE,iCAAiB,IAAI,KAA8B;CAGnD;CACA,YAAY,aAA0B;AACpC,QAAA,cAAoB;;CAEtB,eAAe,SAA6B;AAC1C,SAAO,MAAA,YAAkB,cAAc,QAAQ;;CAIjD,MAAa,KAAsB;EACjC,MAAM,SAAS,IAAI,aAAa,IAAI;AAGpC,UAFY,OAAO,WAAW,EAE9B;GACE,KAAK,GACH,QAAO,KAAK,SAAS,OAAO;GAC9B,KAAK,GACH,QAAO,KAAK,UAAU,OAAO;GAC/B,KAAK,GACH,QAAO,KAAK,QAAQ,OAAO;GAC7B,KAAK,GACH,QAAO,KAAK,YAAY,OAAO;GACjC,KAAK,GACH,QAAO,KAAK,UAAU,OAAO;GAC/B,KAAK,GACH,QAAO,KAAK,UAAU,OAAO;GAC/B,KAAK,GACH,QAAO,KAAK,UAAU,OAAO;GAC/B,KAAK,GACH,QAAO,KAAK,YAAY,OAAO;GACjC,KAAK,GACH,QAAO,KAAK,WAAW,OAAO;GAChC,KAAK,GACH,QAAO,KAAK,UAAU,OAAO;GAC/B,QACE,OAAM,MAAM,2BAA2B;;;CAI7C,SAAiB,QAAoC;AAKnD,SAAO;GACL,KAAK;GACL,WAAW,OAAO,SAAS;GAC3B,YAAY,OAAO,UAAU;GAC7B,KAAK,OAAO,WAAW;GACxB;;CAGH,UAAkB,QAAqC;AACrD,SAAO;GACL,KAAK;GACL,WAAW,OAAO,SAAS;GAC3B,YAAY,OAAO,YAAY;GAChC;;CAGH,QAAgB,QAAmC;EACjD,MAAM,UAAU,OAAO,WAAW;EAClC,MAAM,aAAa,OAAO,YAAY;EACtC,MAAM,WAAW,OAAO,YAAY;AAGpC,OAAK,WAAW,IAAI,SAAS;GAAC;GAAY;GAAS,CAAC;AAEpD,SAAO;GAAC,KAAK;GAAQ;GAAS;GAAY;GAAS;;CAGrD,YAAoB,QAAuC;EAGzD,MAAM,cAAc,OAAO,WAAW;EACtC,MAAM,SAAS,OAAO,YAAY;EAClC,MAAM,OAAO,OAAO,YAAY;EAChC,MAAM,kBAAkB,KAAK,4BAA4B,OAAO;EAChE,MAAM,UAAU,OAAO,MAAM,OAAO,WAAW,QAC7C,KAAK,mBAAmB,OAAO,CAChC;EAGD,MAAM,MAAuB;GAC3B,KAAK;GACL;GACA;GACA;GACA;GACA;GACA,YATiB,QAAQ,QAAO,OAAM,GAAG,QAAQ,EAAI,CAAC,KAAI,OAAM,GAAG,KAAK;GAUzE;AAGD,OAAK,eAAe,IAAI,aAAa,IAAI;AAEzC,SAAO;;CAGT,4BAAoC,QAAsB;EAExD,MAAM,QAAQ,OAAO,WAAW;AAEhC,UAAQ,OAAR;GACE,KAAK,IACH,QAAO;GACT,KAAK,IACH,QAAO;GACT,KAAK,IACH,QAAO;GACT,KAAK,IACH,QAAO;GACT,QACE,OAAM,MAAM,4BAA4B,OAAO,aAAa,MAAM,GAAG;;;CAI3E,mBAA2B,QAAsC;EAC/D,MAAM,QAAQ,OAAO,WAAW;EAChC,MAAM,OAAO,OAAO,YAAY;EAChC,MAAM,UAAU,OAAO,WAAW;AAGlC,SAAO;GACL;GACA;GACA;GACA,SANc,OAAO,WAAW;GAOhC,YAAY;GACZ,UAAU;GACV,GAAG,KAAK,WAAW,IAAI,QAAQ;GAE/B,QAAQ,MAAA,cAAoB,QAAQ;GACrC;;CAGH,UAAkB,QAAqC;EACrD,MAAM,WAAW,KAAK,eAAe,IAAI,OAAO,WAAW,CAAC;AAE5D,MAAI,CAAC,SACH,OAAM,MAAM,mBAAmB;AAGjC,SAAO,WAAW;AAElB,SAAO;GACL,KAAK;GACL;GACA,KAAK,KAAK,UAAU,QAAQ,SAAS;GACtC;;CAGH,UAAkB,QAAqC;EACrD,MAAM,WAAW,KAAK,eAAe,IAAI,OAAO,WAAW,CAAC;AAE5D,MAAI,CAAC,SACH,OAAM,MAAM,mBAAmB;EAGjC,IAAI,MAAkC;EACtC,IAAI,MAAkC;EACtC,IAAI,OAAmC;EACvC,MAAM,YAAY,OAAO,WAAW;AAEpC,MAAI,cAAc,IAAY;AAC5B,SAAM,KAAK,aAAa,QAAQ,SAAS;AACzC,UAAO,WAAW;AAClB,UAAO,KAAK,UAAU,QAAQ,SAAS;aAC9B,cAAc,IAAY;AACnC,SAAM,KAAK,UAAU,QAAQ,SAAS;AACtC,UAAO,WAAW;AAClB,UAAO,KAAK,UAAU,QAAQ,UAAU,IAAI;aACnC,cAAc,GACvB,QAAO,KAAK,UAAU,QAAQ,SAAS;MAEvC,OAAM,MAAM,0BAA0B,OAAO,aAAa,UAAU,GAAG;AAGzE,SAAO;GAAC,KAAK;GAAU;GAAU;GAAK;GAAK,KAAK;GAAK;;CAGvD,UAAkB,QAAqC;EACrD,MAAM,WAAW,KAAK,eAAe,IAAI,OAAO,WAAW,CAAC;AAE5D,MAAI,CAAC,SACH,OAAM,MAAM,mBAAmB;EAGjC,IAAI,MAAkC;EACtC,IAAI,MAAkC;EACtC,MAAM,YAAY,OAAO,WAAW;AAEpC,MAAI,cAAc,GAChB,OAAM,KAAK,aAAa,QAAQ,SAAS;WAChC,cAAc,GACvB,OAAM,KAAK,UAAU,QAAQ,SAAS;MAEtC,OAAM,MAAM,0BAA0B,OAAO,aAAa,UAAU,GAAG;AAGzE,SAAO;GAAC,KAAK;GAAU;GAAU;GAAK;GAAI;;CAG5C,aACE,QACA,UACqB;EACrB,MAAM,QAAQ,KAAK,UAAU,QAAQ,SAAS;EAC9C,MAAM,MAAM,OAAO,OAAO,KAAK;AAE/B,OAAK,MAAM,KAAK,SAAS,WAMvB,KAAI,KAAK,MAAM,OAAO,OAAO,KAAA,IAAY,MAAM;AAGjD,SAAO;;CAGT,UACE,QACA,EAAC,WACD,wBACqB;EACrB,MAAM,UAAU,OAAO,WAAW;EAClC,MAAM,QAAQ,OAAO,OAAO,KAAK;AAEjC,OAAK,IAAI,IAAI,GAAG,IAAI,SAAS,KAAK;GAChC,MAAM,EAAC,MAAM,WAAU,QAAQ;GAC/B,MAAM,OAAO,OAAO,WAAW;AAE/B,WAAQ,MAAR;IACE,KAAK;KACH,MAAM,QAAQ,OAAO,WAAW;AAIhC,WAAM,QAHO,OAAO,KAAK,MAAM;AAI/B;IACF,KAAK;KACH,MAAM,UAAU,OAAO,WAAW;KAClC,MAAM,SAAS,OAAO,KAAK,QAAQ;AAEnC,WAAM,QAAQ,OADE,OAAO,WAAW,OAAO,CACZ;AAC7B;IACF,KAAK;AACH,WAAM,QAAQ;AACd;IACF,KAAK;AACH,WAAM,QAAQ,yBAAyB;AACvC;IACF,QACE,OAAM,MAAM,0BAA0B,OAAO,aAAa,KAAK,GAAG;;;AAIxE,SAAO;;CAGT,YAAoB,QAAuC;EACzD,MAAM,QAAQ,OAAO,WAAW;EAChC,MAAM,QAAQ,OAAO,WAAW;AAEhC,SAAO;GACL,KAAK;GACL,SAAS,QAAQ,QAAQ,EAAI;GAC7B,iBAAiB,QAAQ,QAAQ,EAAK;GACtC,WAAW,OAAO,MAChB,aACM,KAAK,eAAe,IAAI,OAAO,WAAW,CAAC,CAClD;GACF;;CAGH,WAAmB,QAAsC;EACvD,MAAM,QAAQ,OAAO,WAAW;AAEhC,SAAO;GACL,KAAK;GACL;GACA,eAAe,QAAQ,QAAQ,EAAI;GACnC,YAAY,OAAO,SAAS;GAC5B,QAAQ,OAAO,YAAY;GAC3B,SAAS,OAAO,KAAK,OAAO,WAAW,CAAC;GACzC;;CAGH,UAAkB,QAAqC;AACrD,SAAO;GACL,KAAK;GACL,OAAO,OAAO,WAAW;GACzB,WAAW,OAAO,SAAS;GAC3B,cAAc,OAAO,SAAS;GAC9B,YAAY,OAAO,UAAU;GAC9B"}

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n PG_ADMIN_SHUTDOWN,\n PG_OBJECT_IN_USE,\n PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n lsn: bigint,\n Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n queued: number;\n};\n\nexport async function subscribe(\n lc: LogContext,\n db: PostgresDB,\n slot: string,\n publications: string[],\n lsn: bigint,\n retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n applicationName = 'zero-replicator',\n): Promise<{\n messages: SourceWithPendingQueue<StreamMessage>;\n acks: Sink<bigint>;\n}> {\n const session = postgres(\n defu(\n {\n max: 1,\n ['fetch_types']: false, // Necessary for the streaming protocol\n ['idle_timeout']: null,\n ['max_lifetime']: null as unknown as number,\n connection: {\n ['application_name']: applicationName,\n replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n },\n },\n // ParsedOptions are technically compatible with Options, but happen\n // to not be typed that way. The postgres.js author does an equivalent\n // merge of ParsedOptions and Options here:\n // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n db.options as unknown as Options<Record<string, PostgresType>>,\n ),\n );\n\n // Postgres will send keepalives before timing out a wal_sender. It is possible that\n // these keepalives are not received if there is back-pressure in the replication\n // stream. To keep the connection alive, explicitly send keepalives if none have been\n // sent within the last 75% of the wal_sender_timeout.\n //\n // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n const [{walSenderTimeoutMs}] = await session<\n {walSenderTimeoutMs: number}[]\n >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n AS \"walSenderTimeoutMs\" FROM pg_settings\n WHERE name = 'wal_sender_timeout'`.simple();\n const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n lc.info?.(\n `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms`,\n );\n\n const [readable, writable] = await startReplicationStream(\n lc,\n session,\n slot,\n publications,\n lsn,\n retriesIfReplicationSlotActive + 1,\n );\n\n let lastAckTime = Date.now();\n function sendAck(lsn: bigint) {\n writable.write(makeAck(lsn));\n lastAckTime = Date.now();\n }\n\n const livenessTimer = setInterval(() => {\n const now = Date.now();\n if (now - lastAckTime > manualKeepaliveTimeout) {\n sendAck(0n);\n lc.debug?.(`sent manual keepalive`);\n }\n }, manualKeepaliveTimeout / 5);\n\n let destroyed = false;\n const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n const parser = new PgoutputParser(typeParsers);\n const messages = Subscription.create<StreamMessage>({\n cleanup: () => {\n destroyed = true;\n readable.destroyed || readable.destroy();\n clearInterval(livenessTimer);\n return session.end();\n },\n });\n\n readable.once(\n 'close',\n () =>\n // Only log a warning if the stream was not manually closed.\n destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n );\n readable.once(\n 'error',\n e =>\n // Don't log the shutdown signal. This is the expected way for upstream\n // to close the connection (and will be logged downstream).\n (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n lc.warn?.(`error from ${db.options.host}`, e),\n );\n\n pipe({\n source: readable,\n sink: messages,\n parse: buffer => parseStreamMessage(lc, buffer, parser),\n // Allow a small buffer of messages to be queued in the subscription so\n // that the change-source loop can check the queue to determine if more\n // messages are immediately available.\n bufferMessages: 5,\n });\n\n return {\n messages,\n acks: {push: sendAck},\n };\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n // lit() returns 'escaped_name' with surrounding quotes\n // We strip the quotes since the outer quotes are in the template\n return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n lc: LogContext,\n session: postgres.Sql,\n slot: string,\n publications: string[],\n lsn: bigint,\n maxAttempts: number,\n) {\n for (let i = 0; i < maxAttempts; i++) {\n try {\n const stream = session\n .unsafe(\n `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n proto_version '1',\n publication_names '${formatPublicationNames(publications)}',\n messages 'true'\n )`,\n )\n .execute();\n return await Promise.all([stream.readable(), stream.writable()]);\n } catch (e) {\n if (e instanceof postgres.PostgresError) {\n // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n if (e.code === PG_OBJECT_IN_USE) {\n // The freeing up of the replication slot is not transactional;\n // sometimes it takes time for Postgres to consider the slot\n // inactive.\n lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n await sleep(10);\n continue;\n }\n // error: This slot has been invalidated because it exceeded the maximum reserved size.\n // (This is a different manifestation of a slot being invalidated when\n // the wal exceeds the max_slot_wal_keep_size)\n if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n lc.error?.(`error starting replication stream`, e);\n throw new AutoResetSignal(`unable to start replication stream`, {\n cause: e,\n });\n }\n }\n throw e;\n }\n }\n throw new Error(\n `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n );\n}\n\nfunction parseStreamMessage(\n lc: LogContext,\n buffer: Buffer,\n parser: PgoutputParser,\n): StreamMessage | null {\n // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n lc.warn?.('Unknown message', buffer[0]);\n return null;\n }\n const lsn = buffer.readBigUInt64BE(1);\n if (buffer[0] === 0x77) {\n // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n // (Byte 25 is where the WAL data begins)\n return [lsn, parser.parse(buffer.subarray(25))];\n }\n // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n // (Byte 17: shouldRespond)\n const shouldRespond = buffer.readInt8(17) !== 0;\n return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n const x = Buffer.alloc(34);\n x[0] = 'r'.charCodeAt(0);\n x.writeBigInt64BE(lsn, 1);\n x.writeBigInt64BE(lsn, 9);\n x.writeBigInt64BE(lsn, 17);\n x.writeBigInt64BE(microNow, 25);\n return x;\n}\n"],"mappings":";;;;;;;;;;;;;AAmBA,IAAM,6CAA6C;AAcnD,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAIjB;CACD,MAAM,UAAU,SACd,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;GACd;EACF,EAKD,GAAG,QACJ,CACF;CAQD,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,QAAQ;CACjD,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,IAAK;AACpE,IAAG,OACD,uBAAuB,mBAAmB,gDACK,uBAAuB,IACvE;CAED,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,EAClC;CAED,IAAI,cAAc,KAAK,KAAK;CAC5B,SAAS,QAAQ,KAAa;AAC5B,WAAS,MAAM,QAAQ,IAAI,CAAC;AAC5B,gBAAc,KAAK,KAAK;;CAG1B,MAAM,gBAAgB,kBAAkB;AAEtC,MADY,KAAK,KAAK,GACZ,cAAc,wBAAwB;AAC9C,WAAQ,GAAG;AACX,MAAG,QAAQ,wBAAwB;;IAEpC,yBAAyB,EAAE;CAE9B,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eADC,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK,CAAC,CAC1B;CAC9C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;AACb,cAAY;AACZ,WAAS,aAAa,SAAS,SAAS;AACxC,gBAAc,cAAc;AAC5B,SAAO,QAAQ,KAAK;IAEvB,CAAC;AAEF,UAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,OAAO,CAC5E;AACD,UAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,EAAE,CAChD;AAED,MAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU,mBAAmB,IAAI,QAAQ,OAAO;EAIvD,gBAAgB;EACjB,CAAC;AAEF,QAAO;EACL;EACA,MAAM,EAAC,MAAM,SAAQ;EACtB;;;;;;;AAQH,SAAS,uBAAuB,cAAgC;AAG9D,QAAO,aAAa,KAAI,MAAK,IAAI,EAAE,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,KAAK,IAAI;;AAG7D,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;AACA,MAAK,IAAI,IAAI,GAAG,IAAI,aAAa,IAC/B,KAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,KAAK,CAAC,WAAW,WAAW,IAAI,CAAC;;6BAE3C,uBAAuB,aAAa,CAAC;;SAGzD,CACA,SAAS;AACZ,SAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,UAAU,EAAE,OAAO,UAAU,CAAC,CAAC;UACzD,GAAG;AACV,MAAI,aAAa,SAAS,eAAe;AAEvC,OAAI,EAAE,SAAS,kBAAkB;AAI/B,OAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,EAAE,IAAI,EAAE;AAC9C,UAAM,MAAM,GAAG;AACf;;AAKF,OAAI,EAAE,SAAS,qCAAqC;AAClD,OAAG,QAAQ,qCAAqC,EAAE;AAClD,UAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,GACR,CAAC;;;AAGN,QAAM;;AAGV,OAAM,IAAI,MACR,0BAA0B,YAAY,gCACvC;;AAGH,SAAS,mBACP,IACA,QACA,QACsB;AAEtB,KAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;AAC5C,KAAG,OAAO,mBAAmB,OAAO,GAAG;AACvC,SAAO;;CAET,MAAM,MAAM,OAAO,gBAAgB,EAAE;AACrC,KAAI,OAAO,OAAO,IAGhB,QAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,GAAG,CAAC,CAAC;AAKjD,QAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,GAAG,KAAK;EACA,CAAC;;AAIjD,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,KAAK,GAAG,KAAK,IAAI,KAAM,GAAG,EAAE,CAAC,GAAG,OAAO,IAAK;CAEzE,MAAM,IAAI,OAAO,MAAM,GAAG;AAC1B,GAAE,KAAK,IAAI,WAAW,EAAE;AACxB,GAAE,gBAAgB,KAAK,EAAE;AACzB,GAAE,gBAAgB,KAAK,EAAE;AACzB,GAAE,gBAAgB,KAAK,GAAG;AAC1B,GAAE,gBAAgB,UAAU,GAAG;AAC/B,QAAO"}

package/out/zero-cache/src/services/change-source/pg/lsn.js.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"lsn.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/lsn.ts"],"sourcesContent":["import {assert} from '../../../../../shared/src/asserts.ts';\nimport {\n majorVersionToString,\n stateVersionFromString,\n} from '../../../types/state-version.ts';\nimport type {Change} from '../protocol/current/data.ts';\n\n/**\n * Parsing and conversion utilities for the pg_lsn Type, which represents\n * the \"Log Sequence Number\" used as a monotonic progress marker for logical\n * replication from a PostgresSQL database.\n *\n * The LSN is a 64-bit integer represented in logical replication as two\n * hexadecimal numbers (up to 8 digits each) separated by a slash. This is\n * converted to a LexiVersion and used as DB-agnostic version in change log,\n * invalidation index, and row version in the tables of the sync replica.\n */\nexport type LSN = string;\n\nexport type RecordType = Change['tag'];\n\nexport function toBigInt(lsn: LSN): bigint {\n const parts = lsn.split('/');\n assert(parts.length === 2, `Malformed LSN: \"${lsn}\"`);\n const high = BigInt(`0x${parts[0]}`);\n const low = BigInt(`0x${parts[1]}`);\n const val = (high << 32n) + low;\n return val;\n}\n\nexport function toStateVersionString(lsn: LSN): string {\n return majorVersionToString(toBigInt(lsn));\n}\n\n/** The LSN is tracked by the `major` component of the StateVersion. */\nexport function fromStateVersionString(ver: string): LSN {\n const {major} = stateVersionFromString(ver);\n return fromBigInt(major);\n}\n\nexport function fromBigInt(val: bigint): LSN {\n const high = val >> 32n;\n const low = val & 0xffffffffn;\n return `${high.toString(16).toUpperCase()}/${low.toString(16).toUpperCase()}`;\n}\n"],"mappings":";;;AAqBA,SAAgB,SAAS,KAAkB;CACzC,MAAM,QAAQ,IAAI,MAAM,IAAI;AAC5B,QAAO,MAAM,WAAW,GAAG,mBAAmB,IAAI,GAAG;CACrD,MAAM,OAAO,OAAO,KAAK,MAAM,KAAK;CACpC,MAAM,MAAM,OAAO,KAAK,MAAM,KAAK;AAEnC,SADa,QAAQ,OAAO;;AAI9B,SAAgB,qBAAqB,KAAkB;AACrD,QAAO,qBAAqB,SAAS,IAAI,CAAC;;;AAI5C,SAAgB,uBAAuB,KAAkB;CACvD,MAAM,EAAC,UAAS,uBAAuB,IAAI;AAC3C,QAAO,WAAW,MAAM;;AAG1B,SAAgB,WAAW,KAAkB;CAC3C,MAAM,OAAO,OAAO;CACpB,MAAM,MAAM,MAAM;AAClB,QAAO,GAAG,KAAK,SAAS,GAAG,CAAC,aAAa,CAAC,GAAG,IAAI,SAAS,GAAG,CAAC,aAAa"}

package/out/zero-cache/src/services/change-source/pg/replication-slots.js DELETED Viewed

@@ -1,162 +0,0 @@
-import { upstreamSchema } from "../../../types/shards.js";
-import { isPostgresError } from "../../../types/pg.js";
-import { toStateVersionString } from "./lsn.js";
-import { runTx } from "../../../db/run-transaction.js";
-import { createReplica, replicationSlotExpression, replicationSlotPrefix } from "./schema/shard.js";
-import { orTimeout } from "../../../types/timeout.js";
-import postgres from "postgres";
-import { PG_CONFIGURATION_LIMIT_EXCEEDED, PG_INSUFFICIENT_PRIVILEGE } from "@drdgvhbh/postgres-error-codes";
-//#region ../zero-cache/src/services/change-source/pg/replication-slots.ts
-var CREATE_REPLICATION_SLOT_TIMEOUT_MS = 3e4;
-var SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1e3;
-async function createReplicationSlot(lc, session, { slotName, failover, lockTimeout = SERVER_LOCK_TIMEOUT_MS }) {
-	await session.unsafe(`SET lock_timeout = ${lockTimeout}`);
-	const raced = await orTimeout(failover ? session.unsafe(`CREATE_REPLICATION_SLOT "${slotName}" LOGICAL pgoutput (FAILOVER)`) : session.unsafe(`CREATE_REPLICATION_SLOT "${slotName}" LOGICAL pgoutput`), CREATE_REPLICATION_SLOT_TIMEOUT_MS);
-	if (raced === "timed-out") {
-		session.end().catch((e) => lc.warn?.(`Error closing timed out replication slot session`, e));
-		throw new Error(`Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}. Crashing to force a clean restart.`);
-	}
-	const [slot] = raced;
-	lc.info?.(`Created replication slot ${slotName}`, slot);
-	return slot;
-}
-/**
-* Replica and slot creation involves two sessions for proper coordination
-* with other replica management logic:
-*
-* * A normal transaction is started and acquires an advisory lock for
-*   replica slot management. This is the same lock that cleanup logic
-*   acquires before cleaning up replication slots.
-* * With the lock held, a new replication slot is created in a
-*   replication session. The API of CREATE_REPLICATION_SLOT is such
-*   that it cannot be done in a transaction, and cannot be followed by
-*   any writes, or else its snapshot (which is needed for initial sync)
-*   would be invalidated.
-* * Once the slot is created, the slot and replica information are recorded
-*   in the `replicas` table before releasing the lock.
-*
-* This locking ensures that:
-* 1. multiple replication managers attempting to create a replication slot
-*    will not use the same name for the replication slot (which is selected
-*    from a pool of reused names).
-* 2. Running replication managers (which use an earlier replica of a lower
-*    rank) will not delete the new slot during their cleanup logic, since
-*    the slot will belong to a replica of a higher rank.
-*/
-async function createReplicaAndSlot(lc, sql, replicationSession, shard, replicaID, failover) {
-	const lockName = replicationSlotManagementLock(shard);
-	const slotPoolPrefix = replicationSlotPrefix(shard);
-	for (let first = true;; first = false) try {
-		return runTx(sql, async (tx) => {
-			await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;
-			let slotName;
-			const names = await tx`
-          SELECT slot_name as name FROM pg_replication_slots
-            WHERE slot_name LIKE ${slotPoolPrefix + "%"};
-        `.values();
-			const inUse = new Set(names.flat());
-			for (let next = 0;; next++) {
-				const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;
-				if (!inUse.has(candidateName)) {
-					slotName = candidateName;
-					break;
-				}
-			}
-			const slot = await createReplicationSlot(lc, replicationSession, {
-				slotName,
-				failover
-			});
-			await createReplica(tx, shard, replicaID, slot.slot_name, toStateVersionString(slot.consistent_point));
-			return slot;
-		});
-	} catch (e) {
-		if (first && e instanceof postgres.PostgresError) {
-			if (isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {
-				await sql`ALTER ROLE current_user WITH REPLICATION`;
-				lc.info?.(`Added the REPLICATION role to database user`);
-				continue;
-			}
-			if (isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {
-				const dropped = await sql`
-                SELECT slot_name as slot, pg_drop_replication_slot(slot_name)
-                  FROM pg_replication_slots
-                  WHERE slot_name LIKE ${replicationSlotExpression(shard)} AND NOT active`;
-				if (dropped.length) {
-					lc.warn?.(`Dropped inactive replication slots: ${dropped.map(({ slot }) => slot)}`, e);
-					continue;
-				}
-				lc.error?.(`Unable to drop replication slots`, e);
-			}
-		}
-		throw e;
-	}
-}
-/**
-* Deletes "old" replicas (i.e. those with a lower rank than the current)
-* and attempts to drop replication slots that are not associated with any
-* replica.
-*
-* If a slot could not be dropped because there is still an active subscriber,
-* it will be reflected in the `draining` count that is returned. When there
-* are draining slots, the method should be retried until all orphaned slots
-* have been dropped.
-*/
-async function dropOldReplicasAndSlots(lc, sql, shard, beforeRank) {
-	const replicasTable = `${upstreamSchema(shard)}.replicas`;
-	const oldReplicas = await sql`
-    SELECT id, rank::float8, slot, version, "initialSyncContext", "subscriberContext"
-     FROM ${sql(replicasTable)} WHERE rank < ${beforeRank};
-  `;
-	if (oldReplicas.length) {
-		lc.info?.(`Deleting ${oldReplicas.length} old replica(s)`, { oldReplicas });
-		await sql`DELETE FROM ${sql(replicasTable)} WHERE rank < ${beforeRank}`;
-	}
-	const lockName = replicationSlotManagementLock(shard);
-	const slotExpression = replicationSlotExpression(shard);
-	return runTx(sql, async (tx) => {
-		await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;
-		const dropped = await tx`
-      SELECT slot_name as slot, pg_drop_replication_slot(slot_name)
-        FROM pg_replication_slots
-        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot
-        WHERE slot_name LIKE ${slotExpression}
-          AND NOT active
-          AND replica.id IS NULL;
-    `;
-		if (dropped.length) lc.info?.(`dropped inactive replication slots`, { dropped });
-		const remaining = await tx`
-      SELECT slot_name as slot, active_pid as pid, replica.id as id
-        FROM pg_replication_slots
-        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot
-        WHERE slot_name LIKE ${slotExpression};
-    `;
-		if (remaining.length) lc.info?.(`remaining replication slots`, { remaining });
-		let active = 0;
-		let draining = 0;
-		for (const { id } of remaining) if (id === null) draining++;
-		else active++;
-		return {
-			dropped: dropped.length,
-			active,
-			draining
-		};
-	});
-}
-var ALPHABET = "abcdefghijklmnopqrstuvwxyz";
-function slotPoolSuffix(n) {
-	n++;
-	let suffix = "";
-	while (n > 0) {
-		n--;
-		suffix = ALPHABET[n % 26] + suffix;
-		n = Math.floor(n / 26);
-	}
-	return suffix;
-}
-function replicationSlotManagementLock(shard) {
-	return `replication-slot-management:${shard.appID}_${shard.shardNum}`;
-}
-//#endregion
-export { createReplicaAndSlot, createReplicationSlot, dropOldReplicasAndSlots };
-//# sourceMappingURL=replication-slots.js.map

package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import {\n PG_CONFIGURATION_LIMIT_EXCEEDED,\n PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {runTx} from '../../../db/run-transaction';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg';\nimport {upstreamSchema, type ShardID} from '../../../types/shards';\nimport {orTimeout} from '../../../types/timeout';\nimport {toStateVersionString} from './lsn';\nimport {\n createReplica,\n replicationSlotExpression,\n replicationSlotPrefix,\n} from './schema/shard';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n slot_name: string;\n consistent_point: string;\n snapshot_name: string;\n output_plugin: string;\n};\n\nexport type CreateSlotSpec = {\n slotName: string;\n\n // Note: must be false if pgVersion < PG_17. Caller must verify.\n failover?: boolean;\n\n // For overriding in tests.\n lockTimeout?: number;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nconst CREATE_REPLICATION_SLOT_TIMEOUT_MS = 30_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n// so all commands must be sent using sql.unsafe(). This is technically safe\n// because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n lc: LogContext,\n session: postgres.Sql,\n {slotName, failover, lockTimeout = SERVER_LOCK_TIMEOUT_MS}: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n // does NOT apply to replication commands, but lock_timeout does (it governs\n // the heavyweight lock wait inside LockAcquire). Setting it here causes\n // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n // walsender, rather than relying solely on the client-side orTimeout\n // which can leave an orphaned backend.\n //\n // An orphaned walsender is actively harmful: by this point the replication\n // slot has already been created and is pinning WAL retention and catalog_xmin.\n // Worse, the slot is marked `active` (the walsender PID is still alive), so\n // the existing cleanup code (which drops inactive slots on retry) can't\n // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n // fires (~2h default) or the blocking transaction finishes.\n await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n const createSlot = failover\n ? session.unsafe<ReplicationSlot[]>(\n /*sql*/ `CREATE_REPLICATION_SLOT \"${slotName}\" LOGICAL pgoutput (FAILOVER)`,\n )\n : session.unsafe<ReplicationSlot[]>(\n /*sql*/ `CREATE_REPLICATION_SLOT \"${slotName}\" LOGICAL pgoutput`,\n );\n const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n if (raced === 'timed-out') {\n // Create slot can block indefinitely waiting for old transactions. End\n // this connection in the background and fail fast so the process restarts.\n void session\n .end()\n .catch(e =>\n lc.warn?.(`Error closing timed out replication slot session`, e),\n );\n throw new Error(\n `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}. ` +\n `Crashing to force a clean restart.`,\n );\n }\n const [slot] = raced;\n lc.info?.(`Created replication slot ${slotName}`, slot);\n return slot;\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n * replica slot management. This is the same lock that cleanup logic\n * acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n * replication session. The API of CREATE_REPLICATION_SLOT is such\n * that it cannot be done in a transaction, and cannot be followed by\n * any writes, or else its snapshot (which is needed for initial sync)\n * would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n * in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n * will not use the same name for the replication slot (which is selected\n * from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n * rank) will not delete the new slot during their cleanup logic, since\n * the slot will belong to a replica of a higher rank.\n */\nexport async function createReplicaAndSlot(\n lc: LogContext,\n sql: PostgresDB,\n replicationSession: postgres.Sql,\n shard: ShardID,\n replicaID: string,\n failover: boolean,\n): Promise<ReplicationSlot> {\n const lockName = replicationSlotManagementLock(shard);\n const slotPoolPrefix = replicationSlotPrefix(shard);\n for (let first = true; ; first = false) {\n try {\n return runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n // Pick an available slotName from the slotPoolPrefix pool.\n let slotName: string;\n const names = await tx<{name: string}[]> /*sql*/ `\n SELECT slot_name as name FROM pg_replication_slots\n WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n `.values();\n const inUse = new Set(names.flat());\n for (let next = 0; ; next++) {\n const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n if (!inUse.has(candidateName)) {\n slotName = candidateName;\n break;\n }\n }\n\n const slot = await createReplicationSlot(lc, replicationSession, {\n slotName,\n failover,\n });\n\n await createReplica(\n tx,\n shard,\n replicaID,\n slot.slot_name,\n toStateVersionString(slot.consistent_point),\n );\n\n return slot;\n });\n } catch (e) {\n if (first && e instanceof postgres.PostgresError) {\n if (isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n // Some Postgres variants (e.g. Google Cloud SQL) require that\n // the user have the REPLICATION role in order to create a slot.\n // Note that this must be done by the upstreamDB connection, and\n // does not work in the replicationSession itself.\n await sql`ALTER ROLE current_user WITH REPLICATION`;\n lc.info?.(`Added the REPLICATION role to database user`);\n continue;\n }\n if (isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n const slotExpression = replicationSlotExpression(shard);\n\n const dropped = await sql<{slot: string}[]>`\n SELECT slot_name as slot, pg_drop_replication_slot(slot_name)\n FROM pg_replication_slots\n WHERE slot_name LIKE ${slotExpression} AND NOT active`;\n if (dropped.length) {\n lc.warn?.(\n `Dropped inactive replication slots: ${dropped.map(({slot}) => slot)}`,\n e,\n );\n continue;\n }\n lc.error?.(`Unable to drop replication slots`, e);\n }\n }\n throw e;\n }\n }\n}\n\n/**\n * Deletes \"old\" replicas (i.e. those with a lower rank than the current)\n * and attempts to drop replication slots that are not associated with any\n * replica.\n *\n * If a slot could not be dropped because there is still an active subscriber,\n * it will be reflected in the `draining` count that is returned. When there\n * are draining slots, the method should be retried until all orphaned slots\n * have been dropped.\n */\nexport async function dropOldReplicasAndSlots(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n beforeRank: bigint,\n): Promise<{dropped: number; active: number; draining: number}> {\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n const oldReplicas = await sql`\n SELECT id, rank::float8, slot, version, \"initialSyncContext\", \"subscriberContext\"\n FROM ${sql(replicasTable)} WHERE rank < ${beforeRank};\n `;\n if (oldReplicas.length) {\n lc.info?.(`Deleting ${oldReplicas.length} old replica(s)`, {oldReplicas});\n await sql`DELETE FROM ${sql(replicasTable)} WHERE rank < ${beforeRank}`;\n }\n\n // The slot / replica cleanup happens within a transaction while holding\n // the replication slot management lock for this shard, to ensure that no\n // slot that belongs to a newer replica is dropped.\n const lockName = replicationSlotManagementLock(shard);\n const slotExpression = replicationSlotExpression(shard);\n return runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const dropped = await tx /*sql*/ `\n SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name LIKE ${slotExpression} \n AND NOT active\n AND replica.id IS NULL;\n `;\n if (dropped.length) {\n lc.info?.(`dropped inactive replication slots`, {dropped});\n }\n\n const remaining = await tx<\n {slot: string; pid: number | null; id: string | null}[]\n > /*sql*/ `\n SELECT slot_name as slot, active_pid as pid, replica.id as id\n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name LIKE ${slotExpression};\n `;\n if (remaining.length) {\n lc.info?.(`remaining replication slots`, {remaining});\n }\n\n let active = 0;\n let draining = 0;\n for (const {id} of remaining) {\n if (id === null) {\n draining++;\n } else {\n active++;\n }\n }\n\n return {\n dropped: dropped.length,\n active,\n draining,\n };\n });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n n++; // Adjust for 0-based indexing\n\n let suffix = '';\n while (n > 0) {\n n--;\n suffix = ALPHABET[n % 26] + suffix;\n n = Math.floor(n / 26);\n }\n return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;AA2CA,IAAM,qCAAqC;AAK3C,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EAAC,UAAU,UAAU,cAAc,0BACT;AAgB1B,OAAM,QAAQ,OAAO,sBAAsB,cAAc;CASzD,MAAM,QAAQ,MAAM,UAPD,WACf,QAAQ,OACE,4BAA4B,SAAS,+BAC9C,GACD,QAAQ,OACE,4BAA4B,SAAS,oBAC9C,EACqC,mCAAmC;AAC7E,KAAI,UAAU,aAAa;AAGpB,UACF,KAAK,CACL,OAAM,MACL,GAAG,OAAO,oDAAoD,EAAE,CACjE;AACH,QAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,sCAEhG;;CAEH,MAAM,CAAC,QAAQ;AACf,IAAG,OAAO,4BAA4B,YAAY,KAAK;AACvD,QAAO;;;;;;;;;;;;;;;;;;;;;;;;;AA0BT,eAAsB,qBACpB,IACA,KACA,oBACA,OACA,WACA,UAC0B;CAC1B,MAAM,WAAW,8BAA8B,MAAM;CACrD,MAAM,iBAAiB,sBAAsB,MAAM;AACnD,MAAK,IAAI,QAAQ,OAAQ,QAAQ,MAC/B,KAAI;AACF,SAAO,MAAM,KAAK,OAAM,OAAM;AAC5B,SAAM,EAAE,yCAAyC,SAAS;GAG1D,IAAI;GACJ,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,QAAQ;GACV,MAAM,QAAQ,IAAI,IAAI,MAAM,MAAM,CAAC;AACnC,QAAK,IAAI,OAAO,IAAK,QAAQ;IAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,KAAK;AAC9D,QAAI,CAAC,MAAM,IAAI,cAAc,EAAE;AAC7B,gBAAW;AACX;;;GAIJ,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;IAC/D;IACA;IACD,CAAC;AAEF,SAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,qBAAqB,KAAK,iBAAiB,CAC5C;AAED,UAAO;IACP;UACK,GAAG;AACV,MAAI,SAAS,aAAa,SAAS,eAAe;AAChD,OAAI,gBAAgB,GAAG,0BAA0B,EAAE;AAKjD,UAAM,GAAG;AACT,OAAG,OAAO,8CAA8C;AACxD;;AAEF,OAAI,gBAAgB,GAAG,gCAAgC,EAAE;IAGvD,MAAM,UAAU,MAAM,GAAqB;;;yCAFpB,0BAA0B,MAAM,CAKT;AAC9C,QAAI,QAAQ,QAAQ;AAClB,QAAG,OACD,uCAAuC,QAAQ,KAAK,EAAC,WAAU,KAAK,IACpE,EACD;AACD;;AAEF,OAAG,QAAQ,oCAAoC,EAAE;;;AAGrD,QAAM;;;;;;;;;;;;;AAeZ,eAAsB,wBACpB,IACA,KACA,OACA,YAC8D;CAC9D,MAAM,gBAAgB,GAAG,eAAe,MAAM,CAAC;CAC/C,MAAM,cAAc,MAAM,GAAG;;YAEnB,IAAI,cAAc,CAAC,gBAAgB,WAAW;;AAExD,KAAI,YAAY,QAAQ;AACtB,KAAG,OAAO,YAAY,YAAY,OAAO,kBAAkB,EAAC,aAAY,CAAC;AACzE,QAAM,GAAG,eAAe,IAAI,cAAc,CAAC,gBAAgB;;CAM7D,MAAM,WAAW,8BAA8B,MAAM;CACrD,MAAM,iBAAiB,0BAA0B,MAAM;AACvD,QAAO,MAAM,KAAK,OAAM,OAAM;AAC5B,QAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,cAAc,CAAC;+BACP,eAAe;;;;AAI1C,MAAI,QAAQ,OACV,IAAG,OAAO,sCAAsC,EAAC,SAAQ,CAAC;EAG5D,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,cAAc,CAAC;+BACP,eAAe;;AAE1C,MAAI,UAAU,OACZ,IAAG,OAAO,+BAA+B,EAAC,WAAU,CAAC;EAGvD,IAAI,SAAS;EACb,IAAI,WAAW;AACf,OAAK,MAAM,EAAC,QAAO,UACjB,KAAI,OAAO,KACT;MAEA;AAIJ,SAAO;GACL,SAAS,QAAQ;GACjB;GACA;GACD;GACD;;AAGJ,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;AACxC;CAEA,IAAI,SAAS;AACb,QAAO,IAAI,GAAG;AACZ;AACA,WAAS,SAAS,IAAI,MAAM;AAC5B,MAAI,KAAK,MAAM,IAAI,GAAG;;AAExB,QAAO;;AAGT,SAAS,8BAA8B,OAAgB;AACrD,QAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM"}

package/out/zero-cache/src/services/change-source/pg/schema/ddl.js.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"ddl.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/ddl.ts"],"sourcesContent":["import {literal as lit} from 'pg-format';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {upstreamSchema, type ShardConfig} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {publishedSchema, publishedSchemaQuery} from './published.ts';\n\n// Sent in the 'version' tag of \"ddlStart\" and \"ddlUpdate\" event messages.\n// This is used to ensure that the message constructed in the upstream\n// Trigger function is compatible with the code processing it in the zero-cache.\n//\n// Increment this when changing the format of the contents of the \"ddl\" events.\n// This will allow old / incompatible code to detect the change and abort.\nexport const PROTOCOL_VERSION = 1;\n\nconst triggerEvent = v.object({\n context: v.object({query: v.string()}).rest(v.string()),\n});\n\n// All DDL events contain a snapshot of the current tables and indexes that\n// are published / relevant to the shard.\nexport const ddlEventSchema = triggerEvent.extend({\n version: v.literal(PROTOCOL_VERSION),\n schema: publishedSchema,\n event: v.object({tag: v.string()}),\n});\n\n/**\n * A {@link DdlStartEvent} message is emitted before every DDL event, containing\n * the current `schema` and the command `tag`.\n *\n * In most cases, the `DdlStartEvent` itself will not be associated with a\n * schema change, in which case `previousSchema` will be `null`. However, the\n * message is still emitted, both for backwards compatibility and to provide\n * the command `tag` context in case an immediately following `DdlStartEvent`\n * tag is emitted with a schema change (which can happen when another event\n * trigger results in a nested ddl statement).\n *\n * In such cases, the `previousSchema` and `schema` fields of the latter event\n * are used to determine the necessary schema change operations (as they are\n * with `ddlUpdate` and `schemaSnapshot` events), and the `tag` of the\n * preceding start event indicates the command that precipitated the schema\n * change (e.g. a CREATE vs ALTER) to determine whether a backfill is\n * necessary.\n */\nexport const ddlStartEventSchema = ddlEventSchema.extend({\n type: v.literal('ddlStart'),\n // For ddlStart messages, previousSchema is `null` if there was no change\n // in schema detected. The `schema` field is always set to the current\n // schema.\n //\n // In 1.5.0 it is always present, and can be made non-optional when\n // rollback safe.\n previousSchema: publishedSchema.nullable().optional(),\n // For backwards compatibility with previous versions of the trigger,\n // default an absent `event` field with a semantic equivalent. This\n // field override can be removed in a version that is rollback safe\n // with 1.4.0.\n event: v.object({tag: v.string()}).optional(() => ({tag: 'UNKNOWN'})),\n});\n\nexport type DdlStartEvent = v.Infer<typeof ddlStartEventSchema>;\n\n/**\n * A {@link DdlUpdateEvent} is emitted if there was a change in the schema.\n * It always contains `previousSchema` and (current) `schema` fields, leaving\n * it to the receiver to compute the necessary schema change operations.\n */\nexport const ddlUpdateEventSchema = ddlEventSchema.extend({\n type: v.literal('ddlUpdate'),\n // ddlUpdate messages are only emitted if the schema changed, with the\n // `previousSchema` containing the schema before the change.\n //\n // In 1.5.0 it is always set, and can be made non-optional when\n // rollback safe.\n previousSchema: publishedSchema.optional(),\n});\n\nexport type DdlUpdateEvent = v.Infer<typeof ddlUpdateEventSchema>;\n\n/**\n * The `schemaSnapshot` message is a snapshot of a schema taken in response to\n * a `COMMENT ON PUBLICATION` command, which is a hook recognized by zero\n * to manually emit `previousSchema` and `schema` snapshots when a difference\n * is detected. This is a workaround provided to support detection of schema\n * changes from `ALTER PUBLICATION` commands on supabase, which does not fire\n * event triggers for them (https://github.com/supabase/supautils/issues/123).\n *\n * The hook is exercised by trailing the publication change with a\n * `COMMENT ON PUBLICATION` statement, e.g.\n *\n * ```sql\n * BEGIN;\n * ALTER PUBLICATION my_publication ...;\n * COMMENT ON PUBLICATION my_publication IS 'whatever';\n * COMMIT;\n * ```\n *\n * Note that it is fine to invoke `COMMENT ON PUBLICATION` statements\n * on a database that *does* support event triggers on\n * `ALTER PUBLICATION` statements, as it will simply be a no-op.\n */\nexport const schemaSnapshotEventSchema = ddlEventSchema.extend({\n type: v.literal('schemaSnapshot'),\n previousSchema: publishedSchema.optional(),\n});\n\nexport type SchemaSnapshotEvent = v.Infer<typeof schemaSnapshotEventSchema>;\n\nexport const replicationEventSchema = v.union(\n ddlStartEventSchema,\n ddlUpdateEventSchema,\n schemaSnapshotEventSchema,\n);\n\nexport type ReplicationEvent = v.Infer<typeof replicationEventSchema>;\n\n// Creates a function that appends `_{shard-num}` to the input and\n// quotes the result to be a valid identifier.\nfunction append(shardNum: number) {\n return (name: string) => id(name + '_' + String(shardNum));\n}\n\n// pg_advisory_xact_lock key for serializing ddl statements in order to\n// produce correct schema change diffs.\nconst DDL_SERIALIZATION_LOCK = 0x3c6b8468f1bac0b0n;\n\n/**\n * Event trigger functions contain the core logic that are invoked by triggers.\n *\n * Note that although many of these functions can theoretically be parameterized and\n * shared across shards, it is advantageous to keep the functions in each shard\n * isolated from each other in order to avoid the complexity of shared-function\n * versioning.\n *\n * In a sense, shards (and their triggers and functions) should be thought of as\n * execution environments that can be updated at different schedules. If per-shard\n * triggers called into shared functions, we would have to consider versioning the\n * functions when changing their behavior, backwards compatibility, removal of\n * unused versions, etc. (not unlike versioning of npm packages).\n *\n * Instead, we opt for the simplicity and isolation of having each shard\n * completely own (and maintain) the entirety of its trigger/function stack.\n */\nexport function createEventFunctionStatements(shard: ShardConfig) {\n const {appID, shardNum, publications} = shard;\n const schema = id(upstreamSchema(shard)); // e.g. \"{APP_ID}_{SHARD_ID}\"\n return /*sql*/ `\nCREATE SCHEMA IF NOT EXISTS ${schema};\n\nCREATE OR REPLACE FUNCTION ${schema}.get_trigger_context()\nRETURNS record AS $$\nDECLARE\n result record;\nBEGIN\n SELECT COALESCE(current_query(), 'current_query() returned NULL') AS \"query\" into result;\n RETURN result;\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.notice_ignore(reason TEXT, tag TEXT, target record)\nRETURNS void AS $$\nBEGIN\n RAISE NOTICE '${appID}_${shardNum} ignoring % % %', reason, tag, \n COALESCE(row_to_json(target)::text, '');\nEND\n$$ LANGUAGE plpgsql;\n\n\n-- Note: DROP and CREATE to upgrade from v20 to v21 because the\n-- return type has changed. This can be simplified to CREATE OR REPLACE\n-- once 1.5.0 is rollback safe.\nDROP FUNCTION IF EXISTS ${schema}.schema_specs();\nCREATE FUNCTION ${schema}.schema_specs()\nRETURNS JSON \nSTABLE\nAS $$\n ${publishedSchemaQuery(publications)}\n$$ LANGUAGE sql;\n\n\n-- Stores the most recent published schema\nCREATE TABLE IF NOT EXISTS ${schema}.\"publishedSchema\" (\n current JSON,\n exists BOOL PRIMARY KEY DEFAULT true CHECK (exists)\n);\n\nINSERT INTO ${schema}.\"publishedSchema\" (current) VALUES (${schema}.schema_specs())\n ON CONFLICT (exists) DO \n UPDATE SET current = excluded.current;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.update_schemas(event_type text, tag text, target record)\nRETURNS void AS $$\nDECLARE\n prev_schema_specs JSON;\n schema_specs JSON;\n message TEXT;\nBEGIN\n SELECT current FROM ${schema}.\"publishedSchema\" INTO prev_schema_specs;\n SELECT ${schema}.schema_specs() INTO schema_specs;\n \n IF prev_schema_specs::text != schema_specs::text THEN\n UPDATE ${schema}.\"publishedSchema\" SET current = schema_specs;\n ELSIF event_type = 'ddlStart' THEN\n -- ddlStart events are always be emitted to allow the zero-cache\n -- to track the context of the current command tag in the face of\n -- nested event triggers (e.g. start->start->end->end).\n prev_schema_specs = NULL;\n ELSIF event_type = 'ddlUpdate' THEN\n -- TODO: fold 'schemaSnapshot' into this condition too (i.e. make it \"ELSE\")\n -- when 1.5.0 is rollback safe. Until then, noop schemaSnapshots are sent\n -- for compatibility with 1.0.0 ~ 1.4.0.\n PERFORM ${schema}.notice_ignore('noop', tag, target);\n RETURN;\n END IF;\n\n SELECT json_build_object(\n 'type', event_type,\n 'version', ${PROTOCOL_VERSION},\n 'previousSchema', prev_schema_specs,\n 'schema', schema_specs,\n 'event', json_build_object('tag', tag),\n 'context', ${schema}.get_trigger_context()\n ) INTO message;\n\n PERFORM pg_logical_emit_message(true, '${appID}/${shardNum}/ddl', message);\n\n RAISE NOTICE 'Emitted ${appID}_${shardNum} % for % %', event_type, tag, \n COALESCE(row_to_json(target)::text, '');\nEND\n$$ LANGUAGE plpgsql;\n\n\n-- Hook/workaround to manually trigger replication of schema changes on DBs \n-- that do not support/allow event triggers.\nCREATE OR REPLACE FUNCTION ${schema}.update_schemas()\nRETURNS void AS $$\nBEGIN\n PERFORM ${schema}.update_schemas('schemaSnapshot', 'MANUAL', NULL);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_start()\nRETURNS event_trigger AS $$\nDECLARE\n schema_specs JSON;\n message TEXT;\nBEGIN\n -- serialize DDL statements to compute correct schema change diffs\n PERFORM pg_advisory_xact_lock(${DDL_SERIALIZATION_LOCK});\n PERFORM ${schema}.update_schemas('ddlStart', TG_TAG, NULL);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_end()\nRETURNS event_trigger AS $$\nDECLARE\n publications TEXT[];\n target RECORD;\n relevant RECORD;\n schema_specs JSON;\n message TEXT;\n event TEXT;\nBEGIN\n publications := ARRAY[${lit(publications)}];\n\n SELECT objid, object_type, object_identity \n FROM pg_event_trigger_ddl_commands() \n LIMIT 1 INTO target;\n\n -- Filter DDL updates that are not relevant to the shard (i.e. publications) when possible.\n SELECT true INTO relevant;\n\n -- Note: ALTER TABLE statements may *remove* the table from the set of published\n -- tables, and there is no way to determine if the table \"used to be\" in the\n -- set. Thus, all ALTER TABLE statements must produce a ddl update, similar to\n -- any DROP * statement.\n IF (target.object_type = 'table' AND TG_TAG != 'ALTER TABLE') \n OR target.object_type = 'table column' THEN\n SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n INTO relevant;\n\n ELSIF target.object_type = 'index' THEN\n SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n JOIN pg_indexes as ind ON ind.schemaname = ns.nspname AND ind.indexname = c.relname\n JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = ind.tablename\n WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n INTO relevant;\n\n ELSIF target.object_type = 'publication relation' THEN\n SELECT pb.pubname FROM pg_publication_rel AS rel\n JOIN pg_publication AS pb ON pb.oid = rel.prpubid\n WHERE rel.oid = target.objid AND pb.pubname = ANY (publications) \n INTO relevant;\n\n ELSIF target.object_type = 'publication namespace' THEN\n SELECT pb.pubname FROM pg_publication_namespace AS ns\n JOIN pg_publication AS pb ON pb.oid = ns.pnpubid\n WHERE ns.oid = target.objid AND pb.pubname = ANY (publications) \n INTO relevant;\n\n ELSIF target.object_type = 'schema' THEN\n SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n WHERE ns.oid = target.objid AND pb.pubname = ANY (publications)\n INTO relevant;\n\n ELSIF target.object_type = 'publication' THEN\n SELECT 1 WHERE target.object_identity = ANY (publications)\n INTO relevant;\n\n -- no-op CREATE IF NOT EXIST statements\n ELSIF TG_TAG LIKE 'CREATE %' AND target.object_type IS NULL THEN\n relevant := NULL;\n END IF;\n\n IF relevant IS NULL THEN\n PERFORM ${schema}.notice_ignore('irrelevant', TG_TAG, target);\n RETURN;\n END IF;\n\n IF TG_TAG = 'COMMENT' THEN\n -- Only make schemaSnapshots for COMMENT ON PUBLICATION\n IF target.object_type != 'publication' THEN\n PERFORM ${schema}.notice_ignore('irrelevant', TG_TAG, target);\n RETURN;\n END IF;\n PERFORM ${schema}.update_schemas('schemaSnapshot', TG_TAG, target);\n ELSE\n PERFORM ${schema}.update_schemas('ddlUpdate', TG_TAG, target);\n END IF;\n\nEND\n$$ LANGUAGE plpgsql;\n`;\n}\n\n// Exported for testing.\nexport const TAGS = [\n 'CREATE TABLE',\n 'ALTER TABLE',\n 'CREATE INDEX',\n 'DROP TABLE',\n 'DROP INDEX',\n 'ALTER PUBLICATION',\n 'ALTER SCHEMA',\n] as const;\n\nexport function createEventTriggerStatements(shard: ShardConfig) {\n // Better to assert here than get a cryptic syntax error from Postgres.\n assert(shard.publications.length, `shard publications must be non-empty`);\n\n // Unlike functions, which are namespaced in shard-specific schemas,\n // EVENT TRIGGER names are in the global namespace and thus must include\n // the appID and shardNum.\n const {appID, shardNum} = shard;\n const sharded = append(shardNum);\n const schema = id(upstreamSchema(shard));\n\n const triggers = [\n dropEventTriggerStatements(shard.appID, shard.shardNum),\n /*sql*/ `\nCREATE EVENT TRIGGER ${sharded(`${appID}_ddl_start`)}\n ON ddl_command_start\n WHEN TAG IN (${lit(TAGS)})\n EXECUTE PROCEDURE ${schema}.emit_ddl_start();\n\nCREATE EVENT TRIGGER ${sharded(`${appID}_ddl_end`)}\n ON ddl_command_end\n WHEN TAG IN (${lit([...TAGS, 'COMMENT'])})\n EXECUTE PROCEDURE ${schema}.emit_ddl_end();\n`,\n ];\n\n // Drop legacy functions / triggers.\n triggers.push(\n `DROP FUNCTION IF EXISTS ${schema}.emit_ddl_end(text) CASCADE;`,\n `DROP FUNCTION IF EXISTS ${schema}.notice_ignore(text, record);`,\n );\n for (const tag of [...TAGS, 'COMMENT']) {\n const tagID = tag.toLowerCase().replace(' ', '_');\n triggers.push(`DROP FUNCTION IF EXISTS ${schema}.emit_${tagID}() CASCADE;`);\n }\n return triggers.join('');\n}\n\n// Exported for testing.\nexport function dropEventTriggerStatements(\n appID: string,\n shardID: string | number,\n) {\n return /*sql*/ `\n DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_start_${shardID}`)};\n DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_end_${shardID}`)};\n `;\n}\n"],"mappings":";;;;;;AAqBA,IAAa,iBANQ,eAAE,OAAO,EAC5B,SAAS,eAAE,OAAO,EAAC,OAAO,eAAE,QAAQ,EAAC,CAAC,CAAC,KAAK,eAAE,QAAQ,CAAC,EACxD,CAAC,CAIyC,OAAO;CAChD,SAAS,eAAE,QAAA,EAAyB;CACpC,QAAQ;CACR,OAAO,eAAE,OAAO,EAAC,KAAK,eAAE,QAAQ,EAAC,CAAC;CACnC,CAAC;;;;;;;;;;;;;;;;;;;AAoBF,IAAa,sBAAsB,eAAe,OAAO;CACvD,MAAM,eAAE,QAAQ,WAAW;CAO3B,gBAAgB,gBAAgB,UAAU,CAAC,UAAU;CAKrD,OAAO,eAAE,OAAO,EAAC,KAAK,eAAE,QAAQ,EAAC,CAAC,CAAC,gBAAgB,EAAC,KAAK,WAAU,EAAE;CACtE,CAAC;;;;;;AASF,IAAa,uBAAuB,eAAe,OAAO;CACxD,MAAM,eAAE,QAAQ,YAAY;CAM5B,gBAAgB,gBAAgB,UAAU;CAC3C,CAAC;;;;;;;;;;;;;;;;;;;;;;;AA0BF,IAAa,4BAA4B,eAAe,OAAO;CAC7D,MAAM,eAAE,QAAQ,iBAAiB;CACjC,gBAAgB,gBAAgB,UAAU;CAC3C,CAAC;AAIF,IAAa,yBAAyB,eAAE,MACtC,qBACA,sBACA,0BACD;AAMD,SAAS,OAAO,UAAkB;AAChC,SAAQ,SAAiB,GAAG,OAAO,MAAM,OAAO,SAAS,CAAC;;AAK5D,IAAM,yBAAyB;;;;;;;;;;;;;;;;;;AAmB/B,SAAgB,8BAA8B,OAAoB;CAChE,MAAM,EAAC,OAAO,UAAU,iBAAgB;CACxC,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;AACxC,QAAe;8BACa,OAAO;;6BAER,OAAO;;;;;;;;;;;6BAWP,OAAO;;;kBAGlB,MAAM,GAAG,SAAS;;;;;;;;;0BASV,OAAO;kBACf,OAAO;;;;IAIrB,qBAAqB,aAAa,CAAC;;;;;6BAKV,OAAO;;;;;cAKtB,OAAO,uCAAuC,OAAO;;;;;6BAKtC,OAAO;;;;;;;wBAOZ,OAAO;WACpB,OAAO;;;aAGL,OAAO;;;;;;;;;;cAUN,OAAO;;;;;;;;;;iBAUJ,OAAO;;;2CAGmB,MAAM,GAAG,SAAS;;0BAEnC,MAAM,GAAG,SAAS;;;;;;;;6BAQf,OAAO;;;YAGxB,OAAO;;;;;6BAKU,OAAO;;;;;;;kCAOF,uBAAuB;YAC7C,OAAO;;;;;6BAKU,OAAO;;;;;;;;;;0BAUV,QAAI,aAAa,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cA0D9B,OAAO;;;;;;;gBAOL,OAAO;;;cAGT,OAAO;;cAEP,OAAO;;;;;;;AASrB,IAAa,OAAO;CAClB;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAED,SAAgB,6BAA6B,OAAoB;AAE/D,QAAO,MAAM,aAAa,QAAQ,uCAAuC;CAKzE,MAAM,EAAC,OAAO,aAAY;CAC1B,MAAM,UAAU,OAAO,SAAS;CAChC,MAAM,SAAS,GAAG,eAAe,MAAM,CAAC;CAExC,MAAM,WAAW,CACf,2BAA2B,MAAM,OAAO,MAAM,SAAS,EAC/C;uBACW,QAAQ,GAAG,MAAM,YAAY,CAAC;;iBAEpC,QAAI,KAAK,CAAC;sBACL,OAAO;;uBAEN,QAAQ,GAAG,MAAM,UAAU,CAAC;;iBAElC,QAAI,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC;sBACrB,OAAO;EAE1B;AAGD,UAAS,KACP,2BAA2B,OAAO,+BAClC,2BAA2B,OAAO,+BACnC;AACD,MAAK,MAAM,OAAO,CAAC,GAAG,MAAM,UAAU,EAAE;EACtC,MAAM,QAAQ,IAAI,aAAa,CAAC,QAAQ,KAAK,IAAI;AACjD,WAAS,KAAK,2BAA2B,OAAO,QAAQ,MAAM,aAAa;;AAE7E,QAAO,SAAS,KAAK,GAAG;;AAI1B,SAAgB,2BACd,OACA,SACA;AACA,QAAe;mCACkB,GAAG,GAAG,MAAM,aAAa,UAAU,CAAC;mCACpC,GAAG,GAAG,MAAM,WAAW,UAAU,CAAC"}

package/out/zero-cache/src/services/change-source/pg/schema/init.js.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"init.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {\n getVersionHistory,\n runSchemaMigrations,\n type IncrementalMigrationMap,\n type Migration,\n} from '../../../../db/migration.ts';\nimport type {PostgresDB} from '../../../../types/pg.ts';\nimport {upstreamSchema, type ShardConfig} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {decommissionShard} from '../decommission.ts';\nimport {publishedSchema} from './published.ts';\nimport {\n getMutationsTableDefinition,\n legacyReplicationSlot,\n metadataPublicationName,\n setupTablesAndReplication,\n setupTriggers,\n} from './shard.ts';\n\n/**\n * Ensures that a shard is set up for initial sync.\n */\nexport async function ensureShardSchema(\n lc: LogContext,\n db: PostgresDB,\n shard: ShardConfig,\n): Promise<void> {\n const initialSetup: Migration = {\n migrateSchema: (lc, tx) => setupTablesAndReplication(lc, tx, shard),\n minSafeVersion: 1,\n };\n await runSchemaMigrations(\n lc,\n `upstream-shard-${shard.appID}`,\n upstreamSchema(shard),\n db,\n initialSetup,\n // The incremental migration of any existing replicas will be replaced by\n // the incoming replica being synced, so the replicaVersion here is\n // unnecessary.\n getIncrementalMigrations(shard, 'obsolete'),\n );\n}\n\n/**\n * Updates the schema for an existing shard.\n */\nexport async function updateShardSchema(\n lc: LogContext,\n db: PostgresDB,\n shard: ShardConfig,\n replicaVersion: string,\n): Promise<void> {\n await runSchemaMigrations(\n lc,\n `upstream-shard-${shard.appID}`,\n upstreamSchema(shard),\n db,\n {\n // If the expected existing shard is absent, throw an\n // AutoResetSignal to backtrack and initial sync.\n migrateSchema: () => {\n throw new AutoResetSignal(\n `upstream shard ${upstreamSchema(shard)} is not initialized`,\n );\n },\n },\n getIncrementalMigrations(shard, replicaVersion),\n );\n\n // The decommission check is run in updateShardSchema so that it happens\n // after initial sync, and not when the shard schema is initially set up.\n await decommissionLegacyShard(lc, db, shard);\n}\n\nfunction getIncrementalMigrations(\n shard: ShardConfig,\n replicaVersion?: string,\n): IncrementalMigrationMap {\n const shardConfigTable = `${upstreamSchema(shard)}.shardConfig`;\n\n return {\n 4: {\n migrateSchema: () => {\n throw new AutoResetSignal('resetting to upgrade shard schema');\n },\n minSafeVersion: 3,\n },\n\n // v5: changes the upstream schema organization from \"zero_{SHARD_ID}\" to\n // the \"{APP_ID}_0\". An incremental migration indicates that the previous\n // SHARD_ID was \"0\" and the new APP_ID is \"zero\" (i.e. the default values\n // for those options). In this case, the upstream format is identical, and\n // no migration is necessary. However, the version is bumped to v5 to\n // indicate that it was created with the {APP_ID} configuration and should\n // not be decommissioned as a legacy shard.\n\n 6: {\n migrateSchema: async (lc, sql) => {\n assert(\n replicaVersion,\n `replicaVersion is always passed for incremental migrations`,\n );\n await Promise.all([\n sql`\n ALTER TABLE ${sql(shardConfigTable)} ADD \"replicaVersion\" TEXT`,\n sql`\n UPDATE ${sql(shardConfigTable)} SET ${sql({replicaVersion})}`,\n ]);\n lc.info?.(\n `Recorded replicaVersion ${replicaVersion} in upstream shardConfig`,\n );\n },\n },\n\n // Updates the DDL event trigger protocol to v2, and adds support for\n // ALTER SCHEMA x RENAME TO y\n 7: {\n migrateSchema: async (lc, sql) => {\n const [{publications}] = await sql<{publications: string[]}[]>`\n SELECT publications FROM ${sql(shardConfigTable)}`;\n await setupTriggers(lc, sql, {...shard, publications});\n lc.info?.(`Upgraded to v2 event triggers`);\n },\n },\n\n // Adds support for non-disruptive resyncs, which tracks multiple\n // replicas with different slot names.\n 8: {\n migrateSchema: async (lc, sql) => {\n const legacyShardConfigSchema = v.object({\n replicaVersion: v.string().nullable(),\n initialSchema: publishedSchema.nullable(),\n });\n const result = await sql`\n SELECT \"replicaVersion\", \"initialSchema\" FROM ${sql(shardConfigTable)}`;\n assert(\n result.length === 1,\n () => `Expected exactly one shardConfig row, got ${result.length}`,\n );\n const {replicaVersion, initialSchema} = v.parse(\n result[0],\n legacyShardConfigSchema,\n 'passthrough',\n );\n\n await Promise.all([\n sql`\n CREATE TABLE ${sql(upstreamSchema(shard))}.replicas (\n \"slot\" TEXT PRIMARY KEY,\n \"version\" TEXT NOT NULL,\n \"initialSchema\" JSON NOT NULL\n );\n `,\n sql`\n INSERT INTO ${sql(upstreamSchema(shard))}.replicas ${sql({\n slot: legacyReplicationSlot(shard),\n version: replicaVersion,\n initialSchema,\n })}\n `,\n sql`\n ALTER TABLE ${sql(shardConfigTable)} DROP \"replicaVersion\", DROP \"initialSchema\"\n `,\n ]);\n lc.info?.(`Upgraded schema to support non-disruptive resyncs`);\n },\n },\n\n // v9: Fixes field ordering of compound indexes. This incremental migration\n // only fixes indexes resulting from new schema changes. A full resync is\n // required to fix existing indexes.\n //\n // The migration has been subsumed by the identical logic for migrating\n // to v12 (i.e. a trigger upgrade).\n\n // Adds the `mutations` table used to track mutation results.\n 10: {\n migrateSchema: async (lc, sql) => {\n await sql.unsafe(/*sql*/ `\n ${getMutationsTableDefinition(upstreamSchema(shard))}\n ALTER PUBLICATION ${id(metadataPublicationName(shard.appID, shard.shardNum))} ADD TABLE ${id(upstreamSchema(shard))}.\"mutations\";\n `);\n lc.info?.('Upgraded schema with new mutations table');\n },\n },\n\n // v11: Formerly dropped the schemaVersions table, but restored in the v13\n // migration for rollback safety.\n\n // v12: Upgrade DDL trigger to query schemaOID, needed information for auto-backfill.\n // (subsumed by v14)\n\n // Recreates the legacy schemaVersions table that was prematurely dropped\n // in the (former) v11 migration. It needs to remain present for at least one\n // release in order to be rollback safe.\n //\n // TODO: Drop the table once a release that no longer reads the table has\n // been rolled out.\n 13: {\n migrateSchema: async (_, sql) => {\n await sql`\n CREATE TABLE IF NOT EXISTS ${sql(upstreamSchema(shard))}.\"schemaVersions\" (\n \"minSupportedVersion\" INT4,\n \"maxSupportedVersion\" INT4,\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );`;\n await sql`\n INSERT INTO ${sql(upstreamSchema(shard))}.\"schemaVersions\" \n (\"lock\", \"minSupportedVersion\", \"maxSupportedVersion\")\n VALUES (true, 1, 1)\n ON CONFLICT DO NOTHING;\n `;\n },\n },\n\n // v14: Upgrade DDL trigger to log more info to PG logs.\n // (subsumed by v16)\n\n // Add initialSyncContext column to replicas table.\n 15: {\n migrateSchema: async (_, sql) => {\n await sql`\n ALTER TABLE ${sql(upstreamSchema(shard))}.replicas\n ADD COLUMN \"initialSyncContext\" JSON,\n ADD COLUMN \"subscriberContext\" JSON\n `;\n },\n },\n\n // v16: Upgrade DDL trigger to fire on all ALTER TABLE statements\n // to catch the *removal* of a table from the published set.\n // v17 (1.0.0): Upgrade DDL triggers to support the COMMENT ON PUBLICATION hook for\n // working around the lack of event trigger support for PUBLICATION\n // changes in supabase.\n // This also adds forwards-compatible support for hierarchical logical\n // message prefixes and unknown ddl event types.\n // v18: Pure refactoring of event trigger code.\n // v19 (1.4.0): Correctly handle concurrently issued DDL statements.\n // v20 (1.4.0): Handle nested DDL triggers\n // v21 (1.5.0): Handle cross-transaction DDL operations (i.e. concurrent\n // index operations), and support manual invocation of update_schemas().\n\n 22: {\n migrateSchema: async (_, sql) => {\n await sql`\n ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n ALTER \"initialSchema\" DROP NOT NULL;\n `;\n await sql`\n ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n DROP CONSTRAINT replicas_pkey;\n `;\n await sql`\n ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n ADD COLUMN id TEXT PRIMARY KEY DEFAULT replace(gen_random_uuid()::text, '-', '');\n `;\n await sql`\n ALTER TABLE ${sql(upstreamSchema(shard))}.replicas \n ADD COLUMN rank BIGSERIAL;\n `;\n },\n },\n\n // v23 (1.6.0): Event triggers: Handle the case where current_query()\n // returns NULL. Add more logging to debug unexpected messages\n 23: {\n migrateSchema: async (lc, sql) => {\n const [{publications}] = await sql<{publications: string[]}[]>`\n SELECT publications FROM ${sql(shardConfigTable)}`;\n await setupTriggers(lc, sql, {...shard, publications});\n lc.info?.(`Upgraded DDL event triggers`);\n },\n },\n };\n}\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n getIncrementalMigrations({\n appID: 'unused',\n shardNum: 0,\n publications: ['foo'],\n }),\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n\nexport async function decommissionLegacyShard(\n lc: LogContext,\n db: PostgresDB,\n shard: ShardConfig,\n) {\n if (shard.appID !== 'zero') {\n // When migration from non-default shard ids, e.g. \"zero_prod\" => \"prod_0\",\n // clean up the old \"zero_prod\" shard if it is pre-v5. Note that the v5\n // check is important to guard against cleaning up a **new** \"zero_0\" app\n // that coexists with the current App (with app-id === \"0\").\n const versionHistory = await getVersionHistory(db, `zero_${shard.appID}`);\n if (versionHistory !== null && versionHistory.schemaVersion < 5) {\n await decommissionShard(lc, db, 'zero', shard.appID);\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;AA0BA,eAAsB,kBACpB,IACA,IACA,OACe;AAKf,OAAM,oBACJ,IACA,kBAAkB,MAAM,SACxB,eAAe,MAAM,EACrB,IAR8B;EAC9B,gBAAgB,IAAI,OAAO,0BAA0B,IAAI,IAAI,MAAM;EACnE,gBAAgB;EACjB,EAUC,yBAAyB,OAAO,WAAW,CAC5C;;;;;AAMH,eAAsB,kBACpB,IACA,IACA,OACA,gBACe;AACf,OAAM,oBACJ,IACA,kBAAkB,MAAM,SACxB,eAAe,MAAM,EACrB,IACA,EAGE,qBAAqB;AACnB,QAAM,IAAI,gBACR,kBAAkB,eAAe,MAAM,CAAC,qBACzC;IAEJ,EACD,yBAAyB,OAAO,eAAe,CAChD;AAID,OAAM,wBAAwB,IAAI,IAAI,MAAM;;AAG9C,SAAS,yBACP,OACA,gBACyB;CACzB,MAAM,mBAAmB,GAAG,eAAe,MAAM,CAAC;AAElD,QAAO;EACL,GAAG;GACD,qBAAqB;AACnB,UAAM,IAAI,gBAAgB,oCAAoC;;GAEhE,gBAAgB;GACjB;EAUD,GAAG,EACD,eAAe,OAAO,IAAI,QAAQ;AAChC,UACE,gBACA,6DACD;AACD,SAAM,QAAQ,IAAI,CAChB,GAAG;wBACW,IAAI,iBAAiB,CAAC,6BACpC,GAAG;mBACM,IAAI,iBAAiB,CAAC,OAAO,IAAI,EAAC,gBAAe,CAAC,GAC5D,CAAC;AACF,MAAG,OACD,2BAA2B,eAAe,0BAC3C;KAEJ;EAID,GAAG,EACD,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,CAAC,EAAC,kBAAiB,MAAM,GAA+B;qCACjC,IAAI,iBAAiB;AAClD,SAAM,cAAc,IAAI,KAAK;IAAC,GAAG;IAAO;IAAa,CAAC;AACtD,MAAG,OAAO,gCAAgC;KAE7C;EAID,GAAG,EACD,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,0BAA0B,eAAE,OAAO;IACvC,gBAAgB,eAAE,QAAQ,CAAC,UAAU;IACrC,eAAe,gBAAgB,UAAU;IAC1C,CAAC;GACF,MAAM,SAAS,MAAM,GAAG;0DAC0B,IAAI,iBAAiB;AACvE,UACE,OAAO,WAAW,SACZ,6CAA6C,OAAO,SAC3D;GACD,MAAM,EAAC,gBAAgB,kBAAiB,MACtC,OAAO,IACP,yBACA,cACD;AAED,SAAM,QAAQ,IAAI;IAChB,GAAG;yBACY,IAAI,eAAe,MAAM,CAAC,CAAC;;;;;;IAM1C,GAAG;wBACW,IAAI,eAAe,MAAM,CAAC,CAAC,YAAY,IAAI;KACvD,MAAM,sBAAsB,MAAM;KAClC,SAAS;KACT;KACD,CAAC,CAAC;;IAEH,GAAG;wBACW,IAAI,iBAAiB,CAAC;;IAErC,CAAC;AACF,MAAG,OAAO,oDAAoD;KAEjE;EAUD,IAAI,EACF,eAAe,OAAO,IAAI,QAAQ;AAChC,SAAM,IAAI,OAAe;YACrB,4BAA4B,eAAe,MAAM,CAAC,CAAC;8BACjC,GAAG,wBAAwB,MAAM,OAAO,MAAM,SAAS,CAAC,CAAC,aAAa,GAAG,eAAe,MAAM,CAAC,CAAC;UACpH;AACF,MAAG,OAAO,2CAA2C;KAExD;EAcD,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;AAC/B,SAAM,GAAG;uCACsB,IAAI,eAAe,MAAM,CAAC,CAAC;;;;;AAK1D,SAAM,GAAG;wBACO,IAAI,eAAe,MAAM,CAAC,CAAC;;;;;KAM9C;EAMD,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;AAC/B,SAAM,GAAG;wBACO,IAAI,eAAe,MAAM,CAAC,CAAC;;;;KAK9C;EAeD,IAAI,EACF,eAAe,OAAO,GAAG,QAAQ;AAC/B,SAAM,GAAG;wBACO,IAAI,eAAe,MAAM,CAAC,CAAC;;;AAG3C,SAAM,GAAG;wBACO,IAAI,eAAe,MAAM,CAAC,CAAC;;;AAG3C,SAAM,GAAG;wBACO,IAAI,eAAe,MAAM,CAAC,CAAC;;;AAG3C,SAAM,GAAG;wBACO,IAAI,eAAe,MAAM,CAAC,CAAC;;;KAI9C;EAID,IAAI,EACF,eAAe,OAAO,IAAI,QAAQ;GAChC,MAAM,CAAC,EAAC,kBAAiB,MAAM,GAA+B;qCACjC,IAAI,iBAAiB;AAClD,SAAM,cAAc,IAAI,KAAK;IAAC,GAAG;IAAO;IAAa,CAAC;AACtD,MAAG,OAAO,8BAA8B;KAE3C;EACF;;AAImC,OAAO,KAC3C,yBAAyB;CACvB,OAAO;CACP,UAAU;CACV,cAAc,CAAC,MAAM;CACtB,CAAC,CACH,CAAC,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,KAAK,CAAC,EAAE,EAAE;AAE3D,eAAsB,wBACpB,IACA,IACA,OACA;AACA,KAAI,MAAM,UAAU,QAAQ;EAK1B,MAAM,iBAAiB,MAAM,kBAAkB,IAAI,QAAQ,MAAM,QAAQ;AACzE,MAAI,mBAAmB,QAAQ,eAAe,gBAAgB,EAC5D,OAAM,kBAAkB,IAAI,IAAI,QAAQ,MAAM,MAAM"}

package/out/zero-cache/src/services/change-source/pg/schema/published.js.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"published.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/published.ts"],"sourcesContent":["import {literal} from 'pg-format';\nimport type postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../../../shared/src/bigint-json.ts';\nimport {equals} from '../../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {computeZqlSpecsFromLiteSpecs} from '../../../../db/lite-tables.ts';\nimport {\n mapPostgresToLite,\n mapPostgresToLiteIndex,\n} from '../../../../db/pg-to-lite.ts';\nimport {publishedIndexSpec, publishedTableSpec} from '../../../../db/specs.ts';\nimport {liteTableName} from '../../../../types/names.ts';\n\nexport function publishedSchemaQuery(publications: readonly string[]) {\n // Notes:\n // * There's a bug in PG15 in which generated columns are incorrectly\n // included in pg_publication_tables.attnames, (even though the generated\n // column values are not be included in the replication stream).\n // The WHERE condition `attgenerated = ''` fixes this by explicitly excluding\n // generated columns from the list.\n return (\n /*sql*/ `\nWITH published_columns AS (SELECT \n pc.oid::int8 AS \"oid\",\n nspname AS \"schema\",\n pc.relnamespace::int8 AS \"schemaOID\" ,\n pc.relname AS \"name\", \n pc.relreplident AS \"replicaIdentity\",\n attnum AS \"pos\", \n attname AS \"col\", \n pt.typname AS \"type\", \n atttypid::int8 AS \"typeOID\", \n pt.typtype,\n elem_pt.typtype AS \"elemTyptype\",\n NULLIF(atttypmod, -1) AS \"maxLen\", \n attndims \"arrayDims\", \n attnotnull AS \"notNull\",\n pg_get_expr(pd.adbin, pd.adrelid) as \"dflt\",\n NULLIF(ARRAY_POSITION(conkey, attnum), -1) AS \"keyPos\", \n pb.rowfilter as \"rowFilter\",\n pb.pubname as \"publication\"\nFROM pg_attribute\nJOIN pg_class pc ON pc.oid = attrelid\nJOIN pg_namespace pns ON pns.oid = relnamespace\nJOIN pg_type pt ON atttypid = pt.oid\nLEFT JOIN pg_type elem_pt ON elem_pt.oid = pt.typelem\nJOIN pg_publication_tables as pb ON \n pb.schemaname = nspname AND \n pb.tablename = pc.relname AND\n attname = ANY(pb.attnames)\nLEFT JOIN pg_constraint pk ON pk.contype = 'p' AND pk.connamespace = relnamespace AND pk.conrelid = attrelid\nLEFT JOIN pg_attrdef pd ON pd.adrelid = attrelid AND pd.adnum = attnum\nWHERE pb.pubname IN (${literal(publications)}) AND \n (current_setting('server_version_num')::int >= 160000 OR attgenerated = '')\nORDER BY nspname, pc.relname),\n\ntables AS (SELECT json_build_object(\n 'oid', \"oid\",\n 'schema', \"schema\", \n 'schemaOID', \"schemaOID\",\n 'name', \"name\", \n 'replicaIdentity', \"replicaIdentity\",\n 'columns', json_object_agg(\n DISTINCT\n col,\n jsonb_build_object(\n 'pos', \"pos\",\n 'dataType', CASE WHEN \"arrayDims\" = 0 \n THEN \"type\" \n ELSE substring(\"type\" from 2) || repeat('[]', \"arrayDims\") END,\n 'pgTypeClass', \"typtype\",\n 'elemPgTypeClass', \"elemTyptype\",\n 'typeOID', \"typeOID\",\n -- https://stackoverflow.com/a/52376230\n 'characterMaximumLength', CASE WHEN \"typeOID\" = 1043 OR \"typeOID\" = 1042 \n THEN \"maxLen\" - 4 \n ELSE \"maxLen\" END,\n 'notNull', \"notNull\",\n 'dflt', \"dflt\"\n )\n ),\n 'primaryKey', ARRAY( SELECT json_object_keys(\n json_strip_nulls(\n json_object_agg(\n DISTINCT \"col\", \"keyPos\" ORDER BY \"keyPos\"\n )\n )\n )),\n 'publications', json_object_agg(\n DISTINCT \n \"publication\", \n jsonb_build_object('rowFilter', \"rowFilter\")\n )\n) AS \"table\" FROM published_columns \n GROUP BY \"schema\", \"schemaOID\", \"name\", \"oid\", \"replicaIdentity\"),\n ` +\n // Note: pg_attribute contains column names for tables and for indexes.\n // However, the latter does not get updated when a column in a table is\n // renamed.\n //\n // https://www.postgresql.org/message-id/5860814f-c91d-4ab0-b771-ded90d7b9c55%40www.fastmail.com\n //\n // To address this, the pg_attribute rows are looked up for the index's\n // table rather than the index itself, using the pg_index.indkey array\n // to determine the set and order of columns to include.\n //\n // Notes:\n // * The first bit of indoption is 1 for DESC and 0 for ASC:\n // https://github.com/postgres/postgres/blob/4e1fad37872e49a711adad5d9870516e5c71a375/src/include/catalog/pg_index.h#L89\n // * pg_index.indkey is an int2vector which is 0-based instead of 1-based.\n // * The additional check for attgenerated is required for the aforementioned\n // (in publishedTableQuery) bug in PG15 in which generated columns are\n // incorrectly included in pg_publication_tables.attnames\n /*sql*/ `\n indexed_columns AS (SELECT\n pg_indexes.schemaname as \"schema\",\n pg_indexes.tablename as \"tableName\",\n pg_indexes.indexname as \"name\",\n index_column.name as \"col\",\n CASE WHEN pg_index.indoption[index_column.pos-1] & 1 = 1 THEN 'DESC' ELSE 'ASC' END as \"dir\",\n pg_index.indisunique as \"unique\",\n pg_index.indisprimary as \"isPrimaryKey\",\n pg_index.indisreplident as \"isReplicaIdentity\",\n pg_index.indimmediate as \"isImmediate\"\n FROM pg_indexes\n JOIN pg_namespace ON pg_indexes.schemaname = pg_namespace.nspname\n JOIN pg_class pc ON\n pc.relname = pg_indexes.indexname\n AND pc.relnamespace = pg_namespace.oid\n JOIN pg_publication_tables as pb ON \n pb.schemaname = pg_indexes.schemaname AND \n pb.tablename = pg_indexes.tablename\n JOIN pg_index ON pg_index.indexrelid = pc.oid\n JOIN LATERAL (\n SELECT array_agg(attname) as attnames, array_agg(attgenerated != '') as generated FROM pg_attribute\n WHERE attrelid = pg_index.indrelid\n AND attnum = ANY( (pg_index.indkey::smallint[] )[:pg_index.indnkeyatts - 1] )\n ) as indexed ON true\n JOIN LATERAL (\n SELECT pg_attribute.attname as name, col.index_pos as pos\n FROM UNNEST( (pg_index.indkey::smallint[])[:pg_index.indnkeyatts - 1] ) \n WITH ORDINALITY as col(table_pos, index_pos)\n JOIN pg_attribute ON attrelid = pg_index.indrelid AND attnum = col.table_pos\n ) AS index_column ON true\n LEFT JOIN pg_constraint ON pg_constraint.conindid = pc.oid\n WHERE pb.pubname IN (${literal(publications)})\n AND pg_index.indexprs IS NULL\n AND pg_index.indpred IS NULL\n AND (pg_constraint.contype IS NULL OR pg_constraint.contype IN ('p', 'u'))\n AND indexed.attnames <@ pb.attnames\n AND (current_setting('server_version_num')::int >= 160000 OR false = ALL(indexed.generated))\n ORDER BY\n pg_indexes.schemaname,\n pg_indexes.tablename,\n pg_indexes.indexname,\n index_column.pos ASC),\n \n indexes AS (SELECT json_build_object(\n 'schema', \"schema\",\n 'tableName', \"tableName\",\n 'name', \"name\",\n 'unique', \"unique\",\n 'isPrimaryKey', \"isPrimaryKey\",\n 'isReplicaIdentity', \"isReplicaIdentity\",\n 'isImmediate', \"isImmediate\",\n 'columns', json_object_agg(\"col\", \"dir\")\n ) AS index FROM indexed_columns \n GROUP BY \"schema\", \"tableName\", \"name\", \"unique\", \n \"isPrimaryKey\", \"isReplicaIdentity\", \"isImmediate\")\n\n SELECT json_build_object(\n 'tables', COALESCE((SELECT json_agg(\"table\") FROM tables), '[]'::json),\n 'indexes', COALESCE((SELECT json_agg(\"index\") FROM indexes), '[]'::json)\n ) as \"publishedSchema\"\n `\n );\n}\n\nexport const publishedSchema = v\n .object({\n tables: v.array(publishedTableSpec),\n indexes: v.array(publishedIndexSpec),\n })\n .map(({tables, indexes}) => {\n const zqlSpecs = computeZqlSpecsFromLiteSpecs(\n tables.map(t => mapPostgresToLite(t)),\n indexes.map(mapPostgresToLiteIndex),\n {includeBackfillingColumns: true},\n );\n return {\n indexes,\n\n // Denormalize the schema such that each `table` includes the\n // `replicaIdentityColumns` corresponding to the table's\n // replica identity and associated primary key or index.\n tables: tables.map(table => {\n const replicaIdentityColumns: string[] = [];\n switch (table.replicaIdentity) {\n case 'd':\n replicaIdentityColumns.push(...(table.primaryKey ?? []));\n break;\n case 'i':\n replicaIdentityColumns.push(\n ...Object.keys(\n indexes.find(\n ind =>\n ind.schema === table.schema &&\n ind.tableName === table.name &&\n ind.isReplicaIdentity,\n )?.columns ?? {},\n ),\n );\n break;\n case 'f': {\n // For the key columns of replica identity \"full\", use the columns\n // that the replicator/change-processor will end up using\n // (in #getKey()) as the row key.\n const zqlTable = zqlSpecs.get(liteTableName(table));\n // Note: There zql spec will be absent if the table is not synced,\n // e.g. if it has no suitable unique indexes.\n if (zqlTable) {\n replicaIdentityColumns.push(...zqlTable.tableSpec.primaryKey);\n }\n break;\n }\n }\n return {\n ...table,\n replicaIdentityColumns,\n };\n }),\n };\n });\n\nexport type PublishedSchema = v.Infer<typeof publishedSchema>;\n\nexport type PublishedTableWithReplicaIdentity =\n PublishedSchema['tables'][number];\n\nconst publicationSchema = v.object({\n pubname: v.string(),\n pubinsert: v.boolean(),\n pubupdate: v.boolean(),\n pubdelete: v.boolean(),\n pubtruncate: v.boolean(),\n});\n\nconst publicationsResultSchema = v.array(publicationSchema);\n\nexport type PublicationInfo = PublishedSchema & {\n publications: v.Infer<typeof publicationsResultSchema>;\n};\n\n/**\n * Retrieves published tables and columns.\n */\nexport async function getPublicationInfo(\n sql: postgres.Sql,\n publications: string[],\n): Promise<PublicationInfo> {\n const result = await sql.unsafe(/*sql*/ `\n SELECT \n schemaname AS \"schema\",\n tablename AS \"table\", \n json_object_agg(pubname, attnames) AS \"publications\"\n FROM pg_publication_tables pb\n WHERE pb.pubname IN (${literal(publications)})\n GROUP BY schemaname, tablename;\n\n SELECT ${Object.keys(publicationSchema.shape).join(\n ',',\n )} FROM pg_publication pb\n WHERE pb.pubname IN (${literal(publications)})\n ORDER BY pubname;\n\n ${publishedSchemaQuery(publications)};\n`);\n\n // The first query is used to check that tables in multiple publications\n // always publish the same set of columns.\n const publishedColumns = result[0] as {\n schema: string;\n table: string;\n publications: Record<string, string[]>;\n }[];\n for (const {table, publications} of publishedColumns) {\n let expected: Set<string>;\n Object.entries(publications).forEach(([_, columns], i) => {\n const cols = new Set(columns);\n if (i === 0) {\n expected = cols;\n } else if (!equals(expected, cols)) {\n throw new Error(\n `Table ${table} is exported with different columns: [${[\n ...expected,\n ]}] vs [${[...cols]}]`,\n );\n }\n });\n }\n\n assert(\n result[2][0].publishedSchema,\n () => `Invalid publishedSchema result ${BigIntJSON.stringify(result[2])}`,\n );\n\n return {\n publications: v.parse(result[1], publicationsResultSchema),\n ...v.parse(result[2][0].publishedSchema, publishedSchema),\n };\n}\n"],"mappings":";;;;;;;;;;AAcA,SAAgB,qBAAqB,cAAiC;AAOpE,QACU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uBA+BW,QAAQ,aAAa,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2BA6FlB,QAAQ,aAAa,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiCjD,IAAa,kBAAkB,eAC5B,OAAO;CACN,QAAQ,eAAE,MAAM,mBAAmB;CACnC,SAAS,eAAE,MAAM,mBAAmB;CACrC,CAAC,CACD,KAAK,EAAC,QAAQ,cAAa;CAC1B,MAAM,WAAW,6BACf,OAAO,KAAI,MAAK,kBAAkB,EAAE,CAAC,EACrC,QAAQ,IAAI,uBAAuB,EACnC,EAAC,2BAA2B,MAAK,CAClC;AACD,QAAO;EACL;EAKA,QAAQ,OAAO,KAAI,UAAS;GAC1B,MAAM,yBAAmC,EAAE;AAC3C,WAAQ,MAAM,iBAAd;IACE,KAAK;AACH,4BAAuB,KAAK,GAAI,MAAM,cAAc,EAAE,CAAE;AACxD;IACF,KAAK;AACH,4BAAuB,KACrB,GAAG,OAAO,KACR,QAAQ,MACN,QACE,IAAI,WAAW,MAAM,UACrB,IAAI,cAAc,MAAM,QACxB,IAAI,kBACP,EAAE,WAAW,EAAE,CACjB,CACF;AACD;IACF,KAAK,KAAK;KAIR,MAAM,WAAW,SAAS,IAAI,cAAc,MAAM,CAAC;AAGnD,SAAI,SACF,wBAAuB,KAAK,GAAG,SAAS,UAAU,WAAW;AAE/D;;;AAGJ,UAAO;IACL,GAAG;IACH;IACD;IACD;EACH;EACD;AAOJ,IAAM,oBAAoB,eAAE,OAAO;CACjC,SAAS,eAAE,QAAQ;CACnB,WAAW,eAAE,SAAS;CACtB,WAAW,eAAE,SAAS;CACtB,WAAW,eAAE,SAAS;CACtB,aAAa,eAAE,SAAS;CACzB,CAAC;AAEF,IAAM,2BAA2B,eAAE,MAAM,kBAAkB;;;;AAS3D,eAAsB,mBACpB,KACA,cAC0B;CAC1B,MAAM,SAAS,MAAM,IAAI,OAAe;;;;;;2BAMf,QAAQ,aAAa,CAAC;;;WAGtC,OAAO,KAAK,kBAAkB,MAAM,CAAC,KAC5C,IACD,CAAC;2BACuB,QAAQ,aAAa,CAAC;;;IAG7C,qBAAqB,aAAa,CAAC;EACrC;CAIA,MAAM,mBAAmB,OAAO;AAKhC,MAAK,MAAM,EAAC,OAAO,kBAAiB,kBAAkB;EACpD,IAAI;AACJ,SAAO,QAAQ,aAAa,CAAC,SAAS,CAAC,GAAG,UAAU,MAAM;GACxD,MAAM,OAAO,IAAI,IAAI,QAAQ;AAC7B,OAAI,MAAM,EACR,YAAW;YACF,CAAC,OAAO,UAAU,KAAK,CAChC,OAAM,IAAI,MACR,SAAS,MAAM,wCAAwC,CACrD,GAAG,SACJ,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,GACrB;IAEH;;AAGJ,QACE,OAAO,GAAG,GAAG,uBACP,kCAAkC,WAAW,UAAU,OAAO,GAAG,GACxE;AAED,QAAO;EACL,cAAc,MAAQ,OAAO,IAAI,yBAAyB;EAC1D,GAAG,MAAQ,OAAO,GAAG,GAAG,iBAAiB,gBAAgB;EAC1D"}