npm - @rocicorp/zero - Versions diffs - 1.6.0-canary.9 → 1.6.1 - Mend

@rocicorp/zero 1.6.0-canary.9 → 1.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (29) hide show

package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"initial-sync.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"sourcesContent":["import {mkdtemp, rm} from 'node:fs/promises';\nimport {platform, tmpdir} from 'node:os';\nimport {join} from 'node:path';\nimport {Writable} from 'node:stream';\nimport {pipeline} from 'node:stream/promises';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport type {DownloadStatus} from '../../../../../zero-events/src/status.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {\n createLiteIndexStatement,\n createLiteTableStatement,\n} from '../../../db/create.ts';\nimport {listIndexes, listTables} from '../../../db/lite-tables.ts';\nimport * as Mode from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {\n mapPostgresToLite,\n mapPostgresToLiteIndex,\n} from '../../../db/pg-to-lite.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport type {IndexSpec, PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {\n JSON_STRINGIFIED,\n liteValue,\n type LiteValueType,\n} from '../../../types/lite.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport {PG_15, PG_17} from '../../../types/pg-versions.ts';\nimport {\n connectPgClient,\n pgClient,\n type PostgresDB,\n type PostgresTransaction,\n type PostgresValueType,\n} from '../../../types/pg.ts';\nimport {CpuProfiler} from '../../../types/profiler.ts';\nimport type {ShardConfig} from '../../../types/shards.ts';\nimport {ALLOWED_APP_ID_CHARACTERS} from '../../../types/shards.ts';\nimport {id} from '../../../types/sql.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {ColumnMetadataStore} from '../../replicator/schema/column-metadata.ts';\nimport {initReplicationState} from '../../replicator/schema/replication-state.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicaAndSlot} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport {\n dropShard,\n getInternalShardConfig,\n initReplica,\n validatePublications,\n} from './schema/shard.ts';\n\nexport type InitialSyncOptions = {\n tableCopyWorkers: number;\n profileCopy?: boolean \| undefined;\n textCopy?: boolean \| undefined;\n replicationSlotFailover?: boolean \| undefined;\n /*\n When set, run initial sync in \"shadow\" mode for verification: skip all\n * upstream mutations (no replication slot, no addReplica, no dropShard, no\n * slot drop on failure), suppress status events, and optionally sample\n * rows from each table via TABLESAMPLE BERNOULLI + LIMIT. The caller is\n * responsible for providing (and discarding) a throwaway SQLite `tx`.\n /\n shadow?:\n \| {\n /* 0 < rate <= 1. When 1, no TABLESAMPLE clause is added. /\n sampleRate: number;\n /\n LIMIT N cap appended after TABLESAMPLE. Required: shadow sync is\n * for verification only, so every run must commit to a row budget.\n /\n maxRowsPerTable: number;\n }\n \| undefined;\n};\n\n/* Server context to store with the initial sync metadata for debugging. /\nexport type ServerContext = JSONObject;\n\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n syncOptions: InitialSyncOptions,\n context: ServerContext,\n) {\n if (!ALLOWED_APP_ID_CHARACTERS.test(shard.appID)) {\n throw new Error(\n 'The App ID may only consist of lower-case letters, numbers, and the underscore character',\n );\n }\n const {\n tableCopyWorkers,\n profileCopy,\n textCopy = false,\n replicationSlotFailover = false,\n shadow,\n } = syncOptions;\n const copyProfiler = profileCopy ? await CpuProfiler.connect() : null;\n const sql = await connectPgClient(lc, upstreamURI, 'initial-sync');\n // Replication session is only needed to create a replication slot in the\n // real path. In shadow mode we export a snapshot on a normal connection\n // instead, so no replication session is opened.\n const replicationSession = shadow\n ? undefined\n : pgClient(lc, upstreamURI, 'initial-sync-replication-session', {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n });\n\n const replicaID = Date.now().toString();\n let slotName: string \| undefined; // undefined === shadow\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(\n tx,\n shadow ? async () => {} : undefined,\n ).publish(lc, 'Initializing');\n let releaseShadowSnapshot: (() => Promise<void>) \| undefined;\n try {\n const pgVersion = await checkUpstreamConfig(sql);\n\n // In shadow mode we assume the shard is already initialized and just\n // read back the existing publications. `ensurePublishedTables` would\n // otherwise run DDL and potentially call `dropShard`, which must never\n // happen during a shadow run.\n const {publications} = shadow\n ? await getInternalShardConfig(sql, shard)\n : await ensurePublishedTables(lc, sql, shard);\n lc.info?.(`Upstream is setup with publications [${publications}]`);\n\n const {database, host} = sql.options;\n lc.info?.(\n shadow\n ? `acquiring exported snapshot on ${database}@${host} (shadow mode)`\n : `opening replication session to ${database}@${host}`,\n );\n\n let snapshot: string;\n let lsn: string;\n\n if (shadow) {\n const acquired = await acquireExportedSnapshotForShadowSync(\n lc,\n upstreamURI,\n );\n snapshot = acquired.snapshot;\n lsn = acquired.lsn;\n releaseShadowSnapshot = acquired.release;\n } else {\n const slot = await createReplicaAndSlot(\n lc,\n sql,\n must(replicationSession),\n shard,\n replicaID,\n replicationSlotFailover && pgVersion >= PG_17,\n );\n snapshot = slot.snapshot_name;\n lsn = slot.consistent_point;\n slotName = slot.slot_name;\n }\n\n const initialVersion = toStateVersionString(lsn);\n\n initReplicationState(tx, publications, initialVersion, context);\n\n // Run up to MAX_WORKERS to copy of tables at the replication slot's snapshot.\n const start = performance.now();\n // Retrieve the published schema at the consistent_point.\n const published = await runTx(\n sql,\n async tx => {\n await tx.unsafe(/ sql/ `SET TRANSACTION SNAPSHOT '${snapshot}'`);\n return getPublicationInfo(tx, publications);\n },\n {mode: Mode.READONLY},\n );\n // Note: If this throws, initial-sync is aborted.\n validatePublications(lc, published);\n\n // Now that tables have been validated, kick off the copiers.\n const {tables, indexes} = published;\n const numTables = tables.length;\n if (platform() === 'win32' && tableCopyWorkers < numTables) {\n lc.warn?.(\n `Increasing the number of copy workers from ${tableCopyWorkers} to ` +\n `${numTables} to work around a Node/Postgres connection bug`,\n );\n }\n const numWorkers =\n platform() === 'win32'\n ? numTables\n : Math.min(tableCopyWorkers, numTables);\n\n const copyPool = await connectPgClient(\n lc,\n upstreamURI,\n 'initial-sync-copy-worker',\n {\n max: numWorkers,\n ['max_lifetime']: 120 60, // set a long (2h) limit for COPY streaming\n },\n );\n const copiers = startTableCopyWorkers(\n lc,\n copyPool,\n snapshot,\n numWorkers,\n numTables,\n );\n try {\n createLiteTables(tx, tables, initialVersion);\n const sampleRate = shadow?.sampleRate;\n const maxRowsPerTable = shadow?.maxRowsPerTable;\n const downloads = await Promise.all(\n tables.map(spec =>\n copiers.processReadTask((db, lc) =>\n getInitialDownloadState(lc, db, spec, shadow !== undefined),\n ),\n ),\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying ${numTables} upstream tables at version ${initialVersion}`,\n 5000,\n () => ({downloadStatus: downloads.map(({status}) => status)}),\n );\n\n void copyProfiler?.start();\n const rowCounts = await Promise.all(\n downloads.map(table =>\n copiers.processReadTask((db, lc) =>\n copy(\n lc,\n table,\n copyPool,\n db,\n tx,\n textCopy,\n sampleRate,\n maxRowsPerTable,\n ),\n ),\n ),\n );\n void copyProfiler?.stopAndDispose(lc, 'initial-copy');\n copiers.setDone();\n\n const total = rowCounts.reduce(\n (acc, curr) => ({\n rows: acc.rows + curr.rows,\n flushTime: acc.flushTime + curr.flushTime,\n }),\n {rows: 0, flushTime: 0},\n );\n\n statusPublisher.publish(\n lc,\n 'Indexing',\n `Creating ${indexes.length} indexes`,\n 5000,\n );\n const indexStart = performance.now();\n createLiteIndices(tx, indexes);\n const index = performance.now() - indexStart;\n lc.info?.(`Created indexes (${index.toFixed(3)} ms)`);\n\n if (slotName && replicaID) {\n await initReplica(sql, shard, replicaID, published, context);\n } else {\n assert(shadow, 'expected to be in shadow sync if there is no slotName');\n const rowsByTable = new Map<string, number>();\n for (let i = 0; i < downloads.length; i++) {\n rowsByTable.set(downloads[i].status.table, rowCounts[i].rows);\n }\n verifyShadowReplica(lc, tx, published, rowsByTable);\n }\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Synced ${total.rows.toLocaleString()} rows of ${numTables} tables in ${publications} up to ${lsn} ` +\n `(flush: ${total.flushTime.toFixed(3)}, index: ${index.toFixed(3)}, total: ${elapsed.toFixed(3)} ms)`,\n );\n } finally {\n // All meaningful errors are handled at the processReadTask() call site.\n void copyPool.end().catch(e => lc.warn?.(`Error closing copyPool`, e));\n }\n } catch (e) {\n if (slotName) {\n // If initial-sync did not succeed, make a best effort to drop the\n // orphaned replication slot to avoid running out of slots in\n // pathological cases that result in repeated failures.\n lc.warn?.(`dropping replication slot ${slotName}`, e);\n await sql`\n SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n WHERE slot_name = ${slotName};\n `.catch(e => lc.warn?.(`Unable to drop replication slot ${slotName}`, e));\n }\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n if (releaseShadowSnapshot) {\n await releaseShadowSnapshot().catch(e =>\n lc.warn?.(`Error releasing shadow snapshot`, e),\n );\n }\n if (replicationSession) {\n await replicationSession.end();\n }\n await sql.end();\n }\n}\n\nexport type ShadowSyncOptions = {\n sampleRate: number;\n maxRowsPerTable: number;\n /*\n Parent directory for the throwaway SQLite replica. Defaults to the OS\n * tmpdir. Primarily for tests that need to isolate the scratch directory.\n /\n parentDir?: string \| undefined;\n};\n\n/\n Exercises the initial-sync code path against a sample of rows from every\n * published table, writing into a throwaway SQLite database that is deleted\n * when the run ends. Produces zero upstream mutations: no replication slot,\n * no `addReplica`, no `dropShard`, no status events.\n \n Intended to be invoked periodically so that if a customer ever needs a\n * full reset, we have recent confidence that `initialSync` still works.\n * The shard must already be initialized upstream.\n /\nexport async function shadowInitialSync(\n lc: LogContext,\n shard: ShardConfig,\n upstreamURI: string,\n shadow: ShadowSyncOptions,\n context: ServerContext,\n syncOptions?: Pick<InitialSyncOptions, 'textCopy'>,\n): Promise<void> {\n const dir = await mkdtemp(\n join(shadow.parentDir ?? tmpdir(), 'zero-shadow-sync-'),\n );\n const dbPath = join(dir, 'shadow-replica.db');\n const db = new Database(lc, dbPath);\n try {\n await initialSync(\n lc,\n shard,\n db,\n upstreamURI,\n {\n // Shadow sync copies small samples, so one worker is plenty —\n // no reason to burn additional upstream connections.\n tableCopyWorkers: 1,\n textCopy: syncOptions?.textCopy,\n shadow,\n },\n context,\n );\n } finally {\n try {\n db.close();\n } catch (e) {\n lc.warn?.(`Error closing shadow replica db`, e);\n }\n await rm(dir, {recursive: true, force: true}).catch(e =>\n lc.warn?.(`Error cleaning up shadow replica dir ${dir}`, e),\n );\n }\n}\n\nasync function checkUpstreamConfig(sql: PostgresDB) {\n const {walLevel, version} = (\n await sql<{walLevel: string; version: number}[]>`\n SELECT current_setting('wal_level') as \"walLevel\", \n current_setting('server_version_num') as \"version\";\n `\n )[0];\n\n if (walLevel !== 'logical') {\n throw new Error(\n `Postgres must be configured with \"wal_level = logical\" (currently: \"${walLevel})`,\n );\n }\n if (version < PG_15) {\n throw new Error(\n `Must be running Postgres 15 or higher (currently: \"${version}\")`,\n );\n }\n return version;\n}\n\nasync function ensurePublishedTables(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardConfig,\n validate = true,\n): Promise<{publications: string[]}> {\n const {database, host} = sql.options;\n lc.info?.(`Ensuring upstream PUBLICATION on ${database}@${host}`);\n\n await ensureShardSchema(lc, sql, shard);\n const {publications} = await getInternalShardConfig(sql, shard);\n\n if (validate) {\n let valid = false;\n const nonInternalPublications = publications.filter(\n p => !p.startsWith('_'),\n );\n const exists = await sql`\n SELECT pubname FROM pg_publication WHERE pubname IN ${sql(publications)}\n `.values();\n if (exists.length !== publications.length) {\n lc.warn?.(\n `some configured publications [${publications}] are missing: ` +\n `[${exists.flat()}]. resyncing`,\n );\n } else if (\n !equals(new Set(shard.publications), new Set(nonInternalPublications))\n ) {\n lc.warn?.(\n `requested publications [${shard.publications}] differ from previous` +\n `publications [${nonInternalPublications}]. resyncing`,\n );\n } else {\n valid = true;\n }\n if (!valid) {\n await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n return ensurePublishedTables(lc, sql, shard, false);\n }\n }\n return {publications};\n}\n\nfunction startTableCopyWorkers(\n lc: LogContext,\n db: PostgresDB,\n snapshot: string,\n numWorkers: number,\n numTables: number,\n): TransactionPool {\n const {init} = importSnapshot(snapshot);\n const tableCopiers = new TransactionPool(lc, {\n mode: Mode.READONLY,\n init,\n initialWorkers: numWorkers,\n });\n tableCopiers.run(db);\n\n lc.info?.(`Started ${numWorkers} workers to copy ${numTables} tables`);\n\n if (parseInt(process.versions.node) < 22) {\n lc.warn?.(\n `\\n\\n\\n` +\n `Older versions of Node have a bug that results in an unresponsive\\n` +\n `Postgres connection after running certain combinations of COPY commands.\\n` +\n `If initial sync hangs, run zero-cache with Node v22+. This has the additional\\n` +\n `benefit of being consistent with the Node version run in the production container image.` +\n `\\n\\n\\n`,\n );\n }\n return tableCopiers;\n}\n\n/\n Shadow-mode alternative to `createReplicationSlot`: opens a dedicated\n * READ ONLY REPEATABLE READ transaction on a normal connection, exports the\n * snapshot and captures the current WAL LSN, then holds the transaction\n * open until `release()` is called. The held transaction keeps the snapshot\n * importable by the table-copy workers for the duration of the COPY phase.\n \n Idle-in-transaction timeout is disabled locally so the exporter doesn't\n * get killed while workers are still importing.\n /\nasync function acquireExportedSnapshotForShadowSync(\n lc: LogContext,\n upstreamURI: string,\n): Promise<{\n snapshot: string;\n lsn: string;\n release: () => Promise<void>;\n}> {\n const holder = await connectPgClient(\n lc,\n upstreamURI,\n 'shadow-initial-sync-snapshot',\n {\n max: 1,\n },\n );\n const ready = resolver<{snapshot: string; lsn: string}>();\n const release = resolver<void>();\n const held = holder\n .begin(Mode.READONLY, async tx => {\n await tx`SET LOCAL idle_in_transaction_session_timeout = 0`.execute();\n const [row] = await tx<{snapshot: string; lsn: string}[]>`\n SELECT pg_export_snapshot() AS snapshot,\n pg_current_wal_lsn()::text AS lsn`;\n ready.resolve(row);\n await release.promise;\n })\n .catch(e => ready.reject(e));\n\n let snapshot: string;\n let lsn: string;\n try {\n ({snapshot, lsn} = await ready.promise);\n } catch (e) {\n await holder\n .end()\n .catch(err =>\n lc.warn?.(`Error ending shadow snapshot holder after failure`, err),\n );\n throw e;\n }\n lc.info?.(\n `Exported snapshot ${snapshot} at LSN ${lsn} (shadow initial sync)`,\n );\n return {\n snapshot,\n lsn,\n release: async () => {\n release.resolve();\n try {\n await held;\n } catch (e) {\n lc.warn?.(`snapshot holder transaction ended with error`, e);\n }\n await holder.end();\n },\n };\n}\n\nfunction createLiteTables(\n tx: Database,\n tables: PublishedTableSpec[],\n initialVersion: string,\n) {\n // TODO: Figure out how to reuse the ChangeProcessor here to avoid\n // duplicating the ColumnMetadata logic.\n const columnMetadata = must(ColumnMetadataStore.getInstance(tx));\n for (const t of tables) {\n tx.exec(createLiteTableStatement(mapPostgresToLite(t, initialVersion)));\n const tableName = liteTableName(t);\n for (const [colName, colSpec] of Object.entries(t.columns)) {\n columnMetadata.insert(tableName, colName, colSpec);\n }\n }\n}\n\nfunction createLiteIndices(tx: Database, indices: IndexSpec[]) {\n for (const index of indices) {\n tx.exec(createLiteIndexStatement(mapPostgresToLiteIndex(index)));\n }\n}\n\n/\n Runs structural assertions over a just-synced replica and throws if any\n * fail. Only called in shadow mode — a successful return means the replica\n * is schema-complete, row-count consistent, and its column metadata is in\n * sync with its lite schema.\n \n Note: this intentionally does NOT verify ZQL-queryability. Tables that\n * `computeZqlSpecs` drops (no PK / no all-NOT-NULL unique index, unsupported\n * column types, etc.) are silently skipped in production too — there's\n * nothing shadow-specific about them, so failing here would diverge from\n * prod over an upstream-schema condition prod accepts.\n \n Exported for testing.\n /\nexport function verifyShadowReplica(\n lc: LogContext,\n db: Database,\n published: {tables: PublishedTableSpec[]; indexes: IndexSpec[]},\n rowsByTable: ReadonlyMap<string, number>,\n): void {\n const issues: string[] = [];\n let columnsChecked = 0;\n let rowsChecked = 0;\n\n // 1. Schema completeness: every published table exists in the replica\n // with at least the expected column set.\n const liteTables = listTables(db);\n const liteTableByName = new Map(liteTables.map(t => [t.name, t]));\n for (const pt of published.tables) {\n const name = liteTableName(pt);\n const lite = liteTableByName.get(name);\n if (!lite) {\n issues.push(`missing table in replica: ${name}`);\n continue;\n }\n for (const col of Object.keys(pt.columns)) {\n columnsChecked++;\n if (!(col in lite.columns)) {\n issues.push(`column missing in replica table ${name}: ${col}`);\n }\n }\n }\n\n // Every published index exists in the replica.\n const liteIndexNames = new Set(listIndexes(db).map(i => i.name));\n for (const ix of published.indexes) {\n const mapped = mapPostgresToLiteIndex(ix);\n if (!liteIndexNames.has(mapped.name)) {\n issues.push(\n `missing index in replica: ${mapped.name} on ${mapped.tableName}`,\n );\n }\n }\n\n // 2. Row counts: SQLite COUNT() matches the in-memory copy counter.\n for (const [table, expected] of rowsByTable) {\n try {\n const [row] = db\n .prepare(`SELECT COUNT() as count FROM \"${table}\"`)\n .all<{count: number}>();\n if (row.count !== expected) {\n issues.push(\n `row count mismatch for table ${table}: ` +\n `copy counter reported ${expected}, replica has ${row.count}`,\n );\n } else {\n rowsChecked += row.count;\n }\n } catch (e) {\n issues.push(`could not count rows in table ${table}: ${String(e)}`);\n }\n }\n\n // 3. Column metadata: every published column has a _zero.column_metadata row.\n const meta = must(ColumnMetadataStore.getInstance(db));\n for (const pt of published.tables) {\n const name = liteTableName(pt);\n const rows = meta.getTable(name);\n for (const col of Object.keys(pt.columns)) {\n if (!rows.has(col)) {\n issues.push(`missing column_metadata row for ${name}.${col}`);\n }\n }\n }\n\n if (issues.length) {\n throw new Error(\n `Shadow replica verification failed (${issues.length} issue(s)):\\n` +\n issues.map(i => ` - ${i}`).join('\\n'),\n );\n }\n\n lc.info?.(\n `Shadow replica verification passed: ` +\n `${published.tables.length} tables, ` +\n `${published.indexes.length} indexes, ` +\n `${columnsChecked} columns, ` +\n `${rowsChecked.toLocaleString()} rows`,\n );\n}\n\n// Verified empirically that batches of 50 seem to be the sweet spot,\n// similar to the report in https://sqlite.org/forum/forumpost/8878a512d3652655\n//\n// Exported for testing.\nexport const INSERT_BATCH_SIZE = 50;\n\nconst MB = 1024 1024;\nconst MAX_BUFFERED_ROWS = 10_000;\nconst BUFFERED_SIZE_THRESHOLD = 8 * MB;\n\nexport type DownloadStatements = {\n select: string;\n getTotalRows: string;\n getTotalBytes: string;\n};\n\n/*\n Produces ` TABLESAMPLE BERNOULLI(n)` when `sampleRate` is < 1, else `''`.\n * Row-level Bernoulli sampling is used (rather than SYSTEM) because it\n * produces a more uniform sample and, unlike SYSTEM, still returns rows\n * for small tables at low rates.\n /\nfunction tableSampleClause(sampleRate: number \| undefined): string {\n if (sampleRate === undefined \|\| sampleRate >= 1) {\n return '';\n }\n // Round away float noise (e.g. 0.3 100 = 30.000000000000004) while still\n // preserving sub-integer rates like 0.001 (= 0.1%).\n const pct = parseFloat((sampleRate * 100).toFixed(6));\n return /sql/ ` TABLESAMPLE BERNOULLI(${pct})`;\n}\n\nfunction limitClause(maxRowsPerTable: number \| undefined): string {\n return maxRowsPerTable !== undefined\n ? /sql/ ` LIMIT ${maxRowsPerTable}`\n : '';\n}\n\n/*\n Returns the SELECT column expressions for binary COPY, casting columns\n * without a known binary decoder to `::text`.\n /\nexport function makeBinarySelectExprs(\n table: PublishedTableSpec,\n cols: string[],\n): string[] {\n return cols.map(col => {\n const spec = table.columns[col];\n return hasBinaryDecoder(spec) ? id(col) : `${id(col)}::text`;\n });\n}\n\nexport function makeDownloadStatements(\n table: PublishedTableSpec,\n cols: string[],\n sampleRate?: number \| undefined,\n maxRowsPerTable?: number \| undefined,\n selectExprs?: string[] \| undefined,\n): DownloadStatements {\n const filterConditions = Object.values(table.publications)\n .map(({rowFilter}) => rowFilter)\n .filter(f => !!f); // remove nulls\n const where =\n filterConditions.length === 0\n ? ''\n : /sql/ `WHERE ${filterConditions.join(' OR ')}`;\n const sample = tableSampleClause(sampleRate);\n const limit = limitClause(maxRowsPerTable);\n const fromTable = /sql/ `FROM ${id(table.schema)}.${id(table.name)}${sample} ${where}`;\n const select = /sql/ `SELECT ${(selectExprs ?? cols.map(id)).join(',')} ${fromTable}${limit}`;\n if (limit) {\n // With LIMIT, wrap counts/sums in a subquery so they reflect the\n // capped rowset rather than the full (sampled) table.\n const bytesExpr = cols\n .map(col => `COALESCE(pg_column_size(${id(col)}), 0)`)\n .join(' + ');\n return {\n select,\n getTotalRows: /sql/ `SELECT COUNT()::bigint AS \"totalRows\" FROM (SELECT 1 AS _ ${fromTable}${limit}) s`,\n getTotalBytes: /sql/ `SELECT COALESCE(SUM(b), 0)::bigint AS \"totalBytes\" FROM (SELECT (${bytesExpr}) AS b ${fromTable}${limit}) s`,\n };\n }\n const totalBytes = `(${cols.map(col => `SUM(COALESCE(pg_column_size(${id(col)}), 0))`).join(' + ')})`;\n return {\n select,\n getTotalRows: /sql/ `SELECT COUNT() AS \"totalRows\" ${fromTable}`,\n getTotalBytes: /sql/ `SELECT ${totalBytes} AS \"totalBytes\" ${fromTable}`,\n };\n}\n\ntype DownloadState = {\n spec: PublishedTableSpec;\n status: DownloadStatus;\n};\n\n// Exported for testing.\nexport async function getInitialDownloadState(\n lc: LogContext,\n sql: PostgresDB,\n spec: PublishedTableSpec,\n skipTotals: boolean,\n): Promise<DownloadState> {\n const start = performance.now();\n const table = liteTableName(spec);\n const columns = Object.keys(spec.columns);\n if (skipTotals) {\n // Shadow sync suppresses status events, so the pg_class\n // estimates would be computed and thrown away.\n return {\n spec,\n status: {table, columns, rows: 0, totalRows: 0, totalBytes: 0},\n };\n }\n // Use pg_class estimates instead of expensive COUNT() and\n // SUM(pg_column_size(...)) full table scans. The exact values are only\n // used for progress reporting, so estimates are sufficient.\n const qualifiedName = `${id(spec.schema)}.${id(spec.name)}`;\n const estimateResult = await sql<\n {totalRows: number; totalBytes: number}[]\n >`SELECT GREATEST(reltuples, 0)::float8 AS \"totalRows\",\n pg_table_size(oid)::float8 AS \"totalBytes\"\n FROM pg_class\n WHERE oid = ${qualifiedName}::regclass`;\n\n const {totalRows, totalBytes} = estimateResult[0] ?? {\n totalRows: 0,\n totalBytes: 0,\n };\n\n const state: DownloadState = {\n spec,\n status: {\n table,\n columns,\n rows: 0,\n totalRows,\n totalBytes,\n },\n };\n const elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(`Computed initial download state for ${table} (${elapsed} ms)`, {\n state: state.status,\n });\n return state;\n}\n\nfunction copy(\n lc: LogContext,\n {spec: table, status}: DownloadState,\n dbClient: PostgresDB,\n from: PostgresTransaction,\n to: Database,\n textCopy: boolean,\n sampleRate?: number \| undefined,\n maxRowsPerTable?: number \| undefined,\n) {\n if (textCopy) {\n return copyText(\n lc,\n table,\n status,\n dbClient,\n from,\n to,\n sampleRate,\n maxRowsPerTable,\n );\n }\n return copyBinary(lc, table, status, from, to, sampleRate, maxRowsPerTable);\n}\n\nasync function copyBinary(\n lc: LogContext,\n table: PublishedTableSpec,\n status: DownloadStatus,\n from: PostgresTransaction,\n to: Database,\n sampleRate?: number \| undefined,\n maxRowsPerTable?: number \| undefined,\n) {\n const start = performance.now();\n let flushTime = 0;\n\n const tableName = liteTableName(table);\n const orderedColumns = Object.entries(table.columns);\n\n const columnNames = orderedColumns.map(([c]) => c);\n const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n const valuesSql =\n columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n const insertSql = /sql/ `\n INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n const insertStmt = to.prepare(insertSql);\n const insertBatchStmt = to.prepare(\n insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n );\n\n // Build SELECT with ::text casts for columns without a known binary decoder.\n const select = makeDownloadStatements(\n table,\n columnNames,\n sampleRate,\n maxRowsPerTable,\n makeBinarySelectExprs(table, columnNames),\n ).select;\n\n const decoders = orderedColumns.map(([, spec]) =>\n hasBinaryDecoder(spec) ? makeBinaryDecoder(spec) : textCastDecoder,\n );\n\n const valuesPerRow = columnSpecs.length;\n const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;\n\n const pendingValues: LiteValueType[] = Array.from({\n length: MAX_BUFFERED_ROWS * valuesPerRow,\n });\n let pendingRows = 0;\n let pendingSize = 0;\n\n function flush() {\n const start = performance.now();\n const flushedRows = pendingRows;\n const flushedSize = pendingSize;\n\n let l = 0;\n for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n }\n for (; pendingRows > 0; pendingRows--) {\n insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n }\n const flushedValues = flushedRows * valuesPerRow;\n for (let i = 0; i < flushedValues; i++) {\n pendingValues[i] = undefined as unknown as LiteValueType;\n }\n pendingSize = 0;\n status.rows += flushedRows;\n\n const elapsed = performance.now() - start;\n flushTime += elapsed;\n lc.debug?.(\n `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n );\n }\n\n const binaryParser = new BinaryCopyParser();\n let col = 0;\n\n lc.info?.(`Starting binary copy stream of ${tableName}:`, select);\n\n await pipeline(\n await from\n .unsafe(`COPY (${select}) TO STDOUT WITH (FORMAT binary)`)\n .readable(),\n new Writable({\n highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n write(\n chunk: Buffer,\n _encoding: string,\n callback: (error?: Error) => void,\n ) {\n try {\n for (const fieldBuf of binaryParser.parse(chunk)) {\n pendingSize += fieldBuf === null ? 4 : fieldBuf.length;\n pendingValues[pendingRows * valuesPerRow + col] =\n fieldBuf === null ? null : decoders[col](fieldBuf);\n\n if (++col === decoders.length) {\n col = 0;\n if (\n ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow \|\|\n pendingSize >= BUFFERED_SIZE_THRESHOLD\n ) {\n flush();\n }\n }\n }\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n\n final: (callback: (error?: Error) => void) => {\n try {\n flush();\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n }),\n );\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Finished copying ${status.rows} rows into ${tableName} ` +\n `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n );\n return {rows: status.rows, flushTime};\n}\n\nasync function copyText(\n lc: LogContext,\n table: PublishedTableSpec,\n status: DownloadStatus,\n dbClient: PostgresDB,\n from: PostgresTransaction,\n to: Database,\n sampleRate?: number \| undefined,\n maxRowsPerTable?: number \| undefined,\n) {\n const start = performance.now();\n let flushTime = 0;\n\n const tableName = liteTableName(table);\n const orderedColumns = Object.entries(table.columns);\n\n const columnNames = orderedColumns.map(([c]) => c);\n const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n const valuesSql =\n columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n const insertSql = /sql/ `\n INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n const insertStmt = to.prepare(insertSql);\n const insertBatchStmt = to.prepare(\n insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n );\n\n const {select} = makeDownloadStatements(\n table,\n columnNames,\n sampleRate,\n maxRowsPerTable,\n );\n const valuesPerRow = columnSpecs.length;\n const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;\n\n const pendingValues: LiteValueType[] = Array.from({\n length: MAX_BUFFERED_ROWS * valuesPerRow,\n });\n let pendingRows = 0;\n let pendingSize = 0;\n\n function flush() {\n const start = performance.now();\n const flushedRows = pendingRows;\n const flushedSize = pendingSize;\n\n let l = 0;\n for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n }\n for (; pendingRows > 0; pendingRows--) {\n insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n }\n const flushedValues = flushedRows * valuesPerRow;\n for (let i = 0; i < flushedValues; i++) {\n pendingValues[i] = undefined as unknown as LiteValueType;\n }\n pendingSize = 0;\n status.rows += flushedRows;\n\n const elapsed = performance.now() - start;\n flushTime += elapsed;\n lc.debug?.(\n `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n );\n }\n\n lc.info?.(`Starting text copy stream of ${tableName}:`, select);\n const pgParsers = await getTypeParsers(dbClient, {returnJsonAsString: true});\n const parsers = columnSpecs.map(c => {\n const pgParse = pgParsers.getTypeParser(c.typeOID);\n return (val: string) =>\n liteValue(\n pgParse(val) as PostgresValueType,\n c.dataType,\n JSON_STRINGIFIED,\n );\n });\n\n const tsvParser = new TsvParser();\n let col = 0;\n\n await pipeline(\n await from.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n new Writable({\n highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n write(\n chunk: Buffer,\n _encoding: string,\n callback: (error?: Error) => void,\n ) {\n try {\n for (const text of tsvParser.parse(chunk)) {\n pendingSize += text === null ? 4 : text.length;\n pendingValues[pendingRows * valuesPerRow + col] =\n text === null ? null : parsers[col](text);\n\n if (++col === parsers.length) {\n col = 0;\n if (\n ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow \|\|\n pendingSize >= BUFFERED_SIZE_THRESHOLD\n ) {\n flush();\n }\n }\n }\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n\n final: (callback: (error?: Error) => void) => {\n try {\n flush();\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n }),\n );\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Finished copying ${status.rows} rows into ${tableName} ` +\n `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n );\n return {rows: status.rows, flushTime};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8FA,eAAsB,YACpB,IACA,OACA,IACA,aACA,aACA,SACA;AACA,KAAI,CAAC,0BAA0B,KAAK,MAAM,MAAM,CAC9C,OAAM,IAAI,MACR,2FACD;CAEH,MAAM,EACJ,kBACA,aACA,WAAW,OACX,0BAA0B,OAC1B,WACE;CACJ,MAAM,eAAe,cAAc,MAAM,YAAY,SAAS,GAAG;CACjE,MAAM,MAAM,MAAM,gBAAgB,IAAI,aAAa,eAAe;CAIlE,MAAM,qBAAqB,SACvB,KAAA,IACA,SAAS,IAAI,aAAa,oCAAoC;GAC3D,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CAAC;CAEN,MAAM,YAAY,KAAK,KAAK,CAAC,UAAU;CACvC,IAAI;CACJ,MAAM,kBAAkB,2BAA2B,sBACjD,IACA,SAAS,YAAY,KAAK,KAAA,EAC3B,CAAC,QAAQ,IAAI,eAAe;CAC7B,IAAI;AACJ,KAAI;EACF,MAAM,YAAY,MAAM,oBAAoB,IAAI;EAMhD,MAAM,EAAC,iBAAgB,SACnB,MAAM,uBAAuB,KAAK,MAAM,GACxC,MAAM,sBAAsB,IAAI,KAAK,MAAM;AAC/C,KAAG,OAAO,wCAAwC,aAAa,GAAG;EAElE,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,KAAG,OACD,SACI,kCAAkC,SAAS,GAAG,KAAK,kBACnD,kCAAkC,SAAS,GAAG,OACnD;EAED,IAAI;EACJ,IAAI;AAEJ,MAAI,QAAQ;GACV,MAAM,WAAW,MAAM,qCACrB,IACA,YACD;AACD,cAAW,SAAS;AACpB,SAAM,SAAS;AACf,2BAAwB,SAAS;SAC5B;GACL,MAAM,OAAO,MAAM,qBACjB,IACA,KACA,KAAK,mBAAmB,EACxB,OACA,WACA,2BAA2B,aAAA,KAC5B;AACD,cAAW,KAAK;AAChB,SAAM,KAAK;AACX,cAAW,KAAK;;EAGlB,MAAM,iBAAiB,qBAAqB,IAAI;AAEhD,uBAAqB,IAAI,cAAc,gBAAgB,QAAQ;EAG/D,MAAM,QAAQ,YAAY,KAAK;EAE/B,MAAM,YAAY,MAAM,MACtB,KACA,OAAM,OAAM;AACV,SAAM,GAAG,OAAgB,6BAA6B,SAAS,GAAG;AAClE,UAAO,mBAAmB,IAAI,aAAa;KAE7C,EAAC,MAAM,UAAc,CACtB;AAED,uBAAqB,IAAI,UAAU;EAGnC,MAAM,EAAC,QAAQ,YAAW;EAC1B,MAAM,YAAY,OAAO;AACzB,MAAI,UAAU,KAAK,WAAW,mBAAmB,UAC/C,IAAG,OACD,8CAA8C,iBAAiB,MAC1D,UAAU,gDAChB;EAEH,MAAM,aACJ,UAAU,KAAK,UACX,YACA,KAAK,IAAI,kBAAkB,UAAU;EAE3C,MAAM,WAAW,MAAM,gBACrB,IACA,aACA,4BACA;GACE,KAAK;IACJ,iBAAiB;GACnB,CACF;EACD,MAAM,UAAU,sBACd,IACA,UACA,UACA,YACA,UACD;AACD,MAAI;AACF,oBAAiB,IAAI,QAAQ,eAAe;GAC5C,MAAM,aAAa,QAAQ;GAC3B,MAAM,kBAAkB,QAAQ;GAChC,MAAM,YAAY,MAAM,QAAQ,IAC9B,OAAO,KAAI,SACT,QAAQ,iBAAiB,IAAI,OAC3B,wBAAwB,IAAI,IAAI,MAAM,WAAW,KAAA,EAAU,CAC5D,CACF,CACF;AACD,mBAAgB,QACd,IACA,gBACA,WAAW,UAAU,8BAA8B,kBACnD,YACO,EAAC,gBAAgB,UAAU,KAAK,EAAC,aAAY,OAAO,EAAC,EAC7D;AAEI,iBAAc,OAAO;GAC1B,MAAM,YAAY,MAAM,QAAQ,IAC9B,UAAU,KAAI,UACZ,QAAQ,iBAAiB,IAAI,OAC3B,KACE,IACA,OACA,UACA,IACA,IACA,UACA,YACA,gBACD,CACF,CACF,CACF;AACI,iBAAc,eAAe,IAAI,eAAe;AACrD,WAAQ,SAAS;GAEjB,MAAM,QAAQ,UAAU,QACrB,KAAK,UAAU;IACd,MAAM,IAAI,OAAO,KAAK;IACtB,WAAW,IAAI,YAAY,KAAK;IACjC,GACD;IAAC,MAAM;IAAG,WAAW;IAAE,CACxB;AAED,mBAAgB,QACd,IACA,YACA,YAAY,QAAQ,OAAO,WAC3B,IACD;GACD,MAAM,aAAa,YAAY,KAAK;AACpC,qBAAkB,IAAI,QAAQ;GAC9B,MAAM,QAAQ,YAAY,KAAK,GAAG;AAClC,MAAG,OAAO,oBAAoB,MAAM,QAAQ,EAAE,CAAC,MAAM;AAErD,OAAI,YAAY,UACd,OAAM,YAAY,KAAK,OAAO,WAAW,WAAW,QAAQ;QACvD;AACL,WAAO,QAAQ,wDAAwD;IACvE,MAAM,8BAAc,IAAI,KAAqB;AAC7C,SAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,IACpC,aAAY,IAAI,UAAU,GAAG,OAAO,OAAO,UAAU,GAAG,KAAK;AAE/D,wBAAoB,IAAI,IAAI,WAAW,YAAY;;GAGrD,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,MAAG,OACD,UAAU,MAAM,KAAK,gBAAgB,CAAC,WAAW,UAAU,aAAa,aAAa,SAAS,IAAI,WACrF,MAAM,UAAU,QAAQ,EAAE,CAAC,WAAW,MAAM,QAAQ,EAAE,CAAC,WAAW,QAAQ,QAAQ,EAAE,CAAC,MACnG;YACO;AAEH,YAAS,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,0BAA0B,EAAE,CAAC;;UAEjE,GAAG;AACV,MAAI,UAAU;AAIZ,MAAG,OAAO,6BAA6B,YAAY,EAAE;AACrD,SAAM,GAAG;;8BAEe,SAAS;QAC/B,OAAM,MAAK,GAAG,OAAO,mCAAmC,YAAY,EAAE,CAAC;;AAE3E,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;AACtB,MAAI,sBACF,OAAM,uBAAuB,CAAC,OAAM,MAClC,GAAG,OAAO,mCAAmC,EAAE,CAChD;AAEH,MAAI,mBACF,OAAM,mBAAmB,KAAK;AAEhC,QAAM,IAAI,KAAK;;;;;;;;;;;;;AAwBnB,eAAsB,kBACpB,IACA,OACA,aACA,QACA,SACA,aACe;CACf,MAAM,MAAM,MAAM,QAChB,KAAK,OAAO,aAAa,QAAQ,EAAE,oBAAoB,CACxD;CAED,MAAM,KAAK,IAAI,SAAS,IADT,KAAK,KAAK,oBAAoB,CACV;AACnC,KAAI;AACF,QAAM,YACJ,IACA,OACA,IACA,aACA;GAGE,kBAAkB;GAClB,UAAU,aAAa;GACvB;GACD,EACD,QACD;WACO;AACR,MAAI;AACF,MAAG,OAAO;WACH,GAAG;AACV,MAAG,OAAO,mCAAmC,EAAE;;AAEjD,QAAM,GAAG,KAAK;GAAC,WAAW;GAAM,OAAO;GAAK,CAAC,CAAC,OAAM,MAClD,GAAG,OAAO,wCAAwC,OAAO,EAAE,CAC5D;;;AAIL,eAAe,oBAAoB,KAAiB;CAClD,MAAM,EAAC,UAAU,aACf,MAAM,GAA0C;;;KAIhD;AAEF,KAAI,aAAa,UACf,OAAM,IAAI,MACR,uEAAuE,SAAS,GACjF;AAEH,KAAI,UAAA,KACF,OAAM,IAAI,MACR,sDAAsD,QAAQ,IAC/D;AAEH,QAAO;;AAGT,eAAe,sBACb,IACA,KACA,OACA,WAAW,MACwB;CACnC,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,IAAG,OAAO,oCAAoC,SAAS,GAAG,OAAO;AAEjE,OAAM,kBAAkB,IAAI,KAAK,MAAM;CACvC,MAAM,EAAC,iBAAgB,MAAM,uBAAuB,KAAK,MAAM;AAE/D,KAAI,UAAU;EACZ,IAAI,QAAQ;EACZ,MAAM,0BAA0B,aAAa,QAC3C,MAAK,CAAC,EAAE,WAAW,IAAI,CACxB;EACD,MAAM,SAAS,MAAM,GAAG;4DACgC,IAAI,aAAa,CAAC;QACtE,QAAQ;AACZ,MAAI,OAAO,WAAW,aAAa,OACjC,IAAG,OACD,iCAAiC,aAAa,kBACxC,OAAO,MAAM,CAAC,cACrB;WAED,CAAC,OAAO,IAAI,IAAI,MAAM,aAAa,EAAE,IAAI,IAAI,wBAAwB,CAAC,CAEtE,IAAG,OACD,2BAA2B,MAAM,aAAa,sCAC3B,wBAAwB,cAC5C;MAED,SAAQ;AAEV,MAAI,CAAC,OAAO;AACV,SAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,SAAS,CAAC;AACxD,UAAO,sBAAsB,IAAI,KAAK,OAAO,MAAM;;;AAGvD,QAAO,EAAC,cAAa;;AAGvB,SAAS,sBACP,IACA,IACA,UACA,YACA,WACiB;CACjB,MAAM,EAAC,SAAQ,eAAe,SAAS;CACvC,MAAM,eAAe,IAAI,gBAAgB,IAAI;EAC3C,MAAM;EACN;EACA,gBAAgB;EACjB,CAAC;AACF,cAAa,IAAI,GAAG;AAEpB,IAAG,OAAO,WAAW,WAAW,mBAAmB,UAAU,SAAS;AAEtE,KAAI,SAAS,QAAQ,SAAS,KAAK,GAAG,GACpC,IAAG,OACD,mUAMD;AAEH,QAAO;;;;;;;;;;;;AAaT,eAAe,qCACb,IACA,aAKC;CACD,MAAM,SAAS,MAAM,gBACnB,IACA,aACA,gCACA,EACE,KAAK,GACN,CACF;CACD,MAAM,QAAQ,UAA2C;CACzD,MAAM,UAAU,UAAgB;CAChC,MAAM,OAAO,OACV,MAAM,UAAe,OAAM,OAAM;AAChC,QAAM,EAAE,oDAAoD,SAAS;EACrE,MAAM,CAAC,OAAO,MAAM,EAAqC;;;AAGzD,QAAM,QAAQ,IAAI;AAClB,QAAM,QAAQ;GACd,CACD,OAAM,MAAK,MAAM,OAAO,EAAE,CAAC;CAE9B,IAAI;CACJ,IAAI;AACJ,KAAI;AACF,GAAC,CAAC,UAAU,OAAO,MAAM,MAAM;UACxB,GAAG;AACV,QAAM,OACH,KAAK,CACL,OAAM,QACL,GAAG,OAAO,qDAAqD,IAAI,CACpE;AACH,QAAM;;AAER,IAAG,OACD,qBAAqB,SAAS,UAAU,IAAI,wBAC7C;AACD,QAAO;EACL;EACA;EACA,SAAS,YAAY;AACnB,WAAQ,SAAS;AACjB,OAAI;AACF,UAAM;YACC,GAAG;AACV,OAAG,OAAO,gDAAgD,EAAE;;AAE9D,SAAM,OAAO,KAAK;;EAErB;;AAGH,SAAS,iBACP,IACA,QACA,gBACA;CAGA,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,GAAG,CAAC;AAChE,MAAK,MAAM,KAAK,QAAQ;AACtB,KAAG,KAAK,yBAAyB,kBAAkB,GAAG,eAAe,CAAC,CAAC;EACvE,MAAM,YAAY,cAAc,EAAE;AAClC,OAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,EAAE,QAAQ,CACxD,gBAAe,OAAO,WAAW,SAAS,QAAQ;;;AAKxD,SAAS,kBAAkB,IAAc,SAAsB;AAC7D,MAAK,MAAM,SAAS,QAClB,IAAG,KAAK,yBAAyB,uBAAuB,MAAM,CAAC,CAAC;;;;;;;;;;;;;;;;AAkBpE,SAAgB,oBACd,IACA,IACA,WACA,aACM;CACN,MAAM,SAAmB,EAAE;CAC3B,IAAI,iBAAiB;CACrB,IAAI,cAAc;CAIlB,MAAM,aAAa,WAAW,GAAG;CACjC,MAAM,kBAAkB,IAAI,IAAI,WAAW,KAAI,MAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;AACjE,MAAK,MAAM,MAAM,UAAU,QAAQ;EACjC,MAAM,OAAO,cAAc,GAAG;EAC9B,MAAM,OAAO,gBAAgB,IAAI,KAAK;AACtC,MAAI,CAAC,MAAM;AACT,UAAO,KAAK,6BAA6B,OAAO;AAChD;;AAEF,OAAK,MAAM,OAAO,OAAO,KAAK,GAAG,QAAQ,EAAE;AACzC;AACA,OAAI,EAAE,OAAO,KAAK,SAChB,QAAO,KAAK,mCAAmC,KAAK,IAAI,MAAM;;;CAMpE,MAAM,iBAAiB,IAAI,IAAI,YAAY,GAAG,CAAC,KAAI,MAAK,EAAE,KAAK,CAAC;AAChE,MAAK,MAAM,MAAM,UAAU,SAAS;EAClC,MAAM,SAAS,uBAAuB,GAAG;AACzC,MAAI,CAAC,eAAe,IAAI,OAAO,KAAK,CAClC,QAAO,KACL,6BAA6B,OAAO,KAAK,MAAM,OAAO,YACvD;;AAKL,MAAK,MAAM,CAAC,OAAO,aAAa,YAC9B,KAAI;EACF,MAAM,CAAC,OAAO,GACX,QAAQ,kCAAkC,MAAM,GAAG,CACnD,KAAsB;AACzB,MAAI,IAAI,UAAU,SAChB,QAAO,KACL,gCAAgC,MAAM,0BACX,SAAS,gBAAgB,IAAI,QACzD;MAED,gBAAe,IAAI;UAEd,GAAG;AACV,SAAO,KAAK,iCAAiC,MAAM,IAAI,OAAO,EAAE,GAAG;;CAKvE,MAAM,OAAO,KAAK,oBAAoB,YAAY,GAAG,CAAC;AACtD,MAAK,MAAM,MAAM,UAAU,QAAQ;EACjC,MAAM,OAAO,cAAc,GAAG;EAC9B,MAAM,OAAO,KAAK,SAAS,KAAK;AAChC,OAAK,MAAM,OAAO,OAAO,KAAK,GAAG,QAAQ,CACvC,KAAI,CAAC,KAAK,IAAI,IAAI,CAChB,QAAO,KAAK,mCAAmC,KAAK,GAAG,MAAM;;AAKnE,KAAI,OAAO,OACT,OAAM,IAAI,MACR,uCAAuC,OAAO,OAAO,iBACnD,OAAO,KAAI,MAAK,OAAO,IAAI,CAAC,KAAK,KAAK,CACzC;AAGH,IAAG,OACD,uCACK,UAAU,OAAO,OAAO,WACxB,UAAU,QAAQ,OAAO,YACzB,eAAe,YACf,YAAY,gBAAgB,CAAC,OACnC;;AASH,IAAM,KAAK,OAAO;AAClB,IAAM,oBAAoB;AAC1B,IAAM,0BAA0B,IAAI;;;;;;;AAcpC,SAAS,kBAAkB,YAAwC;AACjE,KAAI,eAAe,KAAA,KAAa,cAAc,EAC5C,QAAO;AAKT,QAAe,0BADH,YAAY,aAAa,KAAK,QAAQ,EAAE,CAAC,CACR;;AAG/C,SAAS,YAAY,iBAA6C;AAChE,QAAO,oBAAoB,KAAA,IACf,UAAU,oBAClB;;;;;;AAON,SAAgB,sBACd,OACA,MACU;AACV,QAAO,KAAK,KAAI,QAAO;EACrB,MAAM,OAAO,MAAM,QAAQ;AAC3B,SAAO,iBAAiB,KAAK,GAAG,GAAG,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC;GACrD;;AAGJ,SAAgB,uBACd,OACA,MACA,YACA,iBACA,aACoB;CACpB,MAAM,mBAAmB,OAAO,OAAO,MAAM,aAAa,CACvD,KAAK,EAAC,gBAAe,UAAU,CAC/B,QAAO,MAAK,CAAC,CAAC,EAAE;CACnB,MAAM,QACJ,iBAAiB,WAAW,IACxB,KACQ,SAAS,iBAAiB,KAAK,OAAO;CACpD,MAAM,SAAS,kBAAkB,WAAW;CAC5C,MAAM,QAAQ,YAAY,gBAAgB;CAC1C,MAAM,YAAoB,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,GAAG,OAAO,GAAG;CACjF,MAAM,SAAiB,WAAW,eAAe,KAAK,IAAI,GAAG,EAAE,KAAK,IAAI,CAAC,GAAG,YAAY;AACxF,KAAI,OAAO;EAGT,MAAM,YAAY,KACf,KAAI,QAAO,2BAA2B,GAAG,IAAI,CAAC,OAAO,CACrD,KAAK,MAAM;AACd,SAAO;GACL;GACA,cAAsB,8DAA8D,YAAY,MAAM;GACtG,eAAuB,oEAAoE,UAAU,SAAS,YAAY,MAAM;GACjI;;CAEH,MAAM,aAAa,IAAI,KAAK,KAAI,QAAO,+BAA+B,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;AACnG,QAAO;EACL;EACA,cAAsB,kCAAkC;EACxD,eAAuB,UAAU,WAAW,mBAAmB;EAChE;;AASH,eAAsB,wBACpB,IACA,KACA,MACA,YACwB;CACxB,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,QAAQ,cAAc,KAAK;CACjC,MAAM,UAAU,OAAO,KAAK,KAAK,QAAQ;AACzC,KAAI,WAGF,QAAO;EACL;EACA,QAAQ;GAAC;GAAO;GAAS,MAAM;GAAG,WAAW;GAAG,YAAY;GAAE;EAC/D;CAaH,MAAM,EAAC,WAAW,gBAPK,MAAM,GAE5B;;;kBAHqB,GAAG,GAAG,KAAK,OAAO,CAAC,GAAG,GAAG,KAAK,KAAK,GAM3B,aAEiB,MAAM;EACnD,WAAW;EACX,YAAY;EACb;CAED,MAAM,QAAuB;EAC3B;EACA,QAAQ;GACN;GACA;GACA,MAAM;GACN;GACA;GACD;EACF;CACD,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,IAAG,OAAO,uCAAuC,MAAM,IAAI,QAAQ,OAAO,EACxE,OAAO,MAAM,QACd,CAAC;AACF,QAAO;;AAGT,SAAS,KACP,IACA,EAAC,MAAM,OAAO,UACd,UACA,MACA,IACA,UACA,YACA,iBACA;AACA,KAAI,SACF,QAAO,SACL,IACA,OACA,QACA,UACA,MACA,IACA,YACA,gBACD;AAEH,QAAO,WAAW,IAAI,OAAO,QAAQ,MAAM,IAAI,YAAY,gBAAgB;;AAG7E,eAAe,WACb,IACA,OACA,QACA,MACA,IACA,YACA,iBACA;CACA,MAAM,QAAQ,YAAY,KAAK;CAC/B,IAAI,YAAY;CAEhB,MAAM,YAAY,cAAc,MAAM;CACtC,MAAM,iBAAiB,OAAO,QAAQ,MAAM,QAAQ;CAEpD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,EAAE;CAClD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,UAAU,KAAK;CAC/D,MAAM,mBAAmB,YAAY,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI;CAE9D,MAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,EAAE,CAAC,MAAM;CACzE,MAAM,YAAoB;mBACT,UAAU,KAAK,iBAAiB,WAAW;CAC5D,MAAM,aAAa,GAAG,QAAQ,UAAU;CACxC,MAAM,kBAAkB,GAAG,QACzB,YAAY,IAAI,YAAY,OAAA,GAA6B,CAC1D;CAGD,MAAM,SAAS,uBACb,OACA,aACA,YACA,iBACA,sBAAsB,OAAO,YAAY,CAC1C,CAAC;CAEF,MAAM,WAAW,eAAe,KAAK,GAAG,UACtC,iBAAiB,KAAK,GAAG,kBAAkB,KAAK,GAAG,gBACpD;CAED,MAAM,eAAe,YAAY;CACjC,MAAM,iBAAiB,eAAA;CAEvB,MAAM,gBAAiC,MAAM,KAAK,EAChD,QAAQ,oBAAoB,cAC7B,CAAC;CACF,IAAI,cAAc;CAClB,IAAI,cAAc;CAElB,SAAS,QAAQ;EACf,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,cAAc;EACpB,MAAM,cAAc;EAEpB,IAAI,IAAI;AACR,SAAO,cAAA,IAAiC,eAAA,GACtC,iBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,eAAgB,CAAC;AAEpE,SAAO,cAAc,GAAG,cACtB,YAAW,IAAI,cAAc,MAAM,GAAI,KAAK,aAAc,CAAC;EAE7D,MAAM,gBAAgB,cAAc;AACpC,OAAK,IAAI,IAAI,GAAG,IAAI,eAAe,IACjC,eAAc,KAAK,KAAA;AAErB,gBAAc;AACd,SAAO,QAAQ;EAEf,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,eAAa;AACb,KAAG,QACD,WAAW,YAAY,GAAG,UAAU,SAAS,YAAY,aAAa,QAAQ,QAAQ,EAAE,CAAC,KAC1F;;CAGH,MAAM,eAAe,IAAI,kBAAkB;CAC3C,IAAI,MAAM;AAEV,IAAG,OAAO,kCAAkC,UAAU,IAAI,OAAO;AAEjE,OAAM,WACJ,MAAM,KACH,OAAO,SAAS,OAAO,kCAAkC,CACzD,UAAU,EACb,IAAI,SAAS;EACX,eAAe;EAEf,MACE,OACA,WACA,UACA;AACA,OAAI;AACF,SAAK,MAAM,YAAY,aAAa,MAAM,MAAM,EAAE;AAChD,oBAAe,aAAa,OAAO,IAAI,SAAS;AAChD,mBAAc,cAAc,eAAe,OACzC,aAAa,OAAO,OAAO,SAAS,KAAK,SAAS;AAEpD,SAAI,EAAE,QAAQ,SAAS,QAAQ;AAC7B,YAAM;AACN,UACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,wBAEf,QAAO;;;AAIb,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAI3D,QAAQ,aAAsC;AAC5C,OAAI;AACF,WAAO;AACP,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAG5D,CAAC,CACH;CAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,IAAG,OACD,oBAAoB,OAAO,KAAK,aAAa,UAAU,WAC1C,UAAU,QAAQ,EAAE,CAAC,eAAe,QAAQ,QAAQ,EAAE,CAAC,OACrE;AACD,QAAO;EAAC,MAAM,OAAO;EAAM;EAAU;;AAGvC,eAAe,SACb,IACA,OACA,QACA,UACA,MACA,IACA,YACA,iBACA;CACA,MAAM,QAAQ,YAAY,KAAK;CAC/B,IAAI,YAAY;CAEhB,MAAM,YAAY,cAAc,MAAM;CACtC,MAAM,iBAAiB,OAAO,QAAQ,MAAM,QAAQ;CAEpD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,EAAE;CAClD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,UAAU,KAAK;CAC/D,MAAM,mBAAmB,YAAY,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI;CAE9D,MAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,EAAE,CAAC,MAAM;CACzE,MAAM,YAAoB;mBACT,UAAU,KAAK,iBAAiB,WAAW;CAC5D,MAAM,aAAa,GAAG,QAAQ,UAAU;CACxC,MAAM,kBAAkB,GAAG,QACzB,YAAY,IAAI,YAAY,OAAA,GAA6B,CAC1D;CAED,MAAM,EAAC,WAAU,uBACf,OACA,aACA,YACA,gBACD;CACD,MAAM,eAAe,YAAY;CACjC,MAAM,iBAAiB,eAAA;CAEvB,MAAM,gBAAiC,MAAM,KAAK,EAChD,QAAQ,oBAAoB,cAC7B,CAAC;CACF,IAAI,cAAc;CAClB,IAAI,cAAc;CAElB,SAAS,QAAQ;EACf,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,cAAc;EACpB,MAAM,cAAc;EAEpB,IAAI,IAAI;AACR,SAAO,cAAA,IAAiC,eAAA,GACtC,iBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,eAAgB,CAAC;AAEpE,SAAO,cAAc,GAAG,cACtB,YAAW,IAAI,cAAc,MAAM,GAAI,KAAK,aAAc,CAAC;EAE7D,MAAM,gBAAgB,cAAc;AACpC,OAAK,IAAI,IAAI,GAAG,IAAI,eAAe,IACjC,eAAc,KAAK,KAAA;AAErB,gBAAc;AACd,SAAO,QAAQ;EAEf,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,eAAa;AACb,KAAG,QACD,WAAW,YAAY,GAAG,UAAU,SAAS,YAAY,aAAa,QAAQ,QAAQ,EAAE,CAAC,KAC1F;;AAGH,IAAG,OAAO,gCAAgC,UAAU,IAAI,OAAO;CAC/D,MAAM,YAAY,MAAM,eAAe,UAAU,EAAC,oBAAoB,MAAK,CAAC;CAC5E,MAAM,UAAU,YAAY,KAAI,MAAK;EACnC,MAAM,UAAU,UAAU,cAAc,EAAE,QAAQ;AAClD,UAAQ,QACN,UACE,QAAQ,IAAI,EACZ,EAAE,UAAA,IAEH;GACH;CAEF,MAAM,YAAY,IAAI,WAAW;CACjC,IAAI,MAAM;AAEV,OAAM,WACJ,MAAM,KAAK,OAAO,SAAS,OAAO,aAAa,CAAC,UAAU,EAC1D,IAAI,SAAS;EACX,eAAe;EAEf,MACE,OACA,WACA,UACA;AACA,OAAI;AACF,SAAK,MAAM,QAAQ,UAAU,MAAM,MAAM,EAAE;AACzC,oBAAe,SAAS,OAAO,IAAI,KAAK;AACxC,mBAAc,cAAc,eAAe,OACzC,SAAS,OAAO,OAAO,QAAQ,KAAK,KAAK;AAE3C,SAAI,EAAE,QAAQ,QAAQ,QAAQ;AAC5B,YAAM;AACN,UACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,wBAEf,QAAO;;;AAIb,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAI3D,QAAQ,aAAsC;AAC5C,OAAI;AACF,WAAO;AACP,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAG5D,CAAC,CACH;CAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,IAAG,OACD,oBAAoB,OAAO,KAAK,aAAa,UAAU,WAC1C,UAAU,QAAQ,EAAE,CAAC,eAAe,QAAQ,QAAQ,EAAE,CAAC,OACrE;AACD,QAAO;EAAC,MAAM,OAAO;EAAM;EAAU"}
1	+ {"version":3,"file":"initial-sync.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"sourcesContent":["import {mkdtemp, rm} from 'node:fs/promises';\nimport {platform, tmpdir} from 'node:os';\nimport {join} from 'node:path';\nimport {Writable} from 'node:stream';\nimport {pipeline} from 'node:stream/promises';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport type {DownloadStatus} from '../../../../../zero-events/src/status.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {\n createLiteIndexStatement,\n createLiteTableStatement,\n} from '../../../db/create.ts';\nimport {listIndexes, listTables} from '../../../db/lite-tables.ts';\nimport * as Mode from '../../../db/mode-enum.ts';\nimport {\n BinaryCopyParser,\n hasBinaryDecoder,\n makeBinaryDecoder,\n textCastDecoder,\n} from '../../../db/pg-copy-binary.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {\n mapPostgresToLite,\n mapPostgresToLiteIndex,\n} from '../../../db/pg-to-lite.ts';\nimport {getTypeParsers} from '../../../db/pg-type-parser.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport type {IndexSpec, PublishedTableSpec} from '../../../db/specs.ts';\nimport {importSnapshot, TransactionPool} from '../../../db/transaction-pool.ts';\nimport {\n JSON_STRINGIFIED,\n liteValue,\n type LiteValueType,\n} from '../../../types/lite.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport {PG_15, PG_17} from '../../../types/pg-versions.ts';\nimport {\n connectPgClient,\n pgClient,\n type PostgresDB,\n type PostgresTransaction,\n type PostgresValueType,\n} from '../../../types/pg.ts';\nimport {CpuProfiler} from '../../../types/profiler.ts';\nimport type {ShardConfig} from '../../../types/shards.ts';\nimport {ALLOWED_APP_ID_CHARACTERS} from '../../../types/shards.ts';\nimport {id} from '../../../types/sql.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {ColumnMetadataStore} from '../../replicator/schema/column-metadata.ts';\nimport {initReplicationState} from '../../replicator/schema/replication-state.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {createReplicaAndSlot} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport {\n dropShard,\n getInternalShardConfig,\n initReplica,\n validatePublications,\n} from './schema/shard.ts';\n\nexport type InitialSyncOptions = {\n tableCopyWorkers: number;\n profileCopy?: boolean \| undefined;\n textCopy?: boolean \| undefined;\n replicationSlotFailover?: boolean \| undefined;\n /*\n When set, run initial sync in \"shadow\" mode for verification: skip all\n * upstream mutations (no replication slot, no addReplica, no dropShard, no\n * slot drop on failure), suppress status events, and optionally sample\n * rows from each table via TABLESAMPLE BERNOULLI + LIMIT. The caller is\n * responsible for providing (and discarding) a throwaway SQLite `tx`.\n /\n shadow?:\n \| {\n /* 0 < rate <= 1. When 1, no TABLESAMPLE clause is added. /\n sampleRate: number;\n /\n LIMIT N cap appended after TABLESAMPLE. Required: shadow sync is\n * for verification only, so every run must commit to a row budget.\n /\n maxRowsPerTable: number;\n }\n \| undefined;\n};\n\n/* Server context to store with the initial sync metadata for debugging. /\nexport type ServerContext = JSONObject;\n\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n syncOptions: InitialSyncOptions,\n context: ServerContext,\n) {\n if (!ALLOWED_APP_ID_CHARACTERS.test(shard.appID)) {\n throw new Error(\n 'The App ID may only consist of lower-case letters, numbers, and the underscore character',\n );\n }\n const {\n tableCopyWorkers,\n profileCopy,\n textCopy = false,\n replicationSlotFailover = false,\n shadow,\n } = syncOptions;\n const copyProfiler = profileCopy ? await CpuProfiler.connect() : null;\n const sql = await connectPgClient(lc, upstreamURI, 'initial-sync');\n // Replication session is only needed to create a replication slot in the\n // real path. In shadow mode we export a snapshot on a normal connection\n // instead, so no replication session is opened.\n const replicationSession = shadow\n ? undefined\n : pgClient(lc, upstreamURI, 'initial-sync-replication-session', {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n });\n\n const replicaID = Date.now().toString();\n let slotName: string \| undefined; // undefined === shadow\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(\n tx,\n shadow ? async () => {} : undefined,\n ).publish(lc, 'Initializing');\n let releaseShadowSnapshot: (() => Promise<void>) \| undefined;\n try {\n const pgVersion = await checkUpstreamConfig(sql);\n\n // In shadow mode we assume the shard is already initialized and just\n // read back the existing publications. `ensurePublishedTables` would\n // otherwise run DDL and potentially call `dropShard`, which must never\n // happen during a shadow run.\n const {publications} = shadow\n ? await getInternalShardConfig(sql, shard)\n : await ensurePublishedTables(lc, sql, shard);\n lc.info?.(`Upstream is setup with publications [${publications}]`);\n\n const {database, host} = sql.options;\n lc.info?.(\n shadow\n ? `acquiring exported snapshot on ${database}@${host} (shadow mode)`\n : `opening replication session to ${database}@${host}`,\n );\n\n let snapshot: string;\n let lsn: string;\n\n if (shadow) {\n const acquired = await acquireExportedSnapshotForShadowSync(\n lc,\n upstreamURI,\n );\n snapshot = acquired.snapshot;\n lsn = acquired.lsn;\n releaseShadowSnapshot = acquired.release;\n } else {\n const slot = await createReplicaAndSlot(\n lc,\n sql,\n must(replicationSession),\n shard,\n replicaID,\n replicationSlotFailover && pgVersion >= PG_17,\n );\n snapshot = slot.snapshot_name;\n lsn = slot.consistent_point;\n slotName = slot.slot_name;\n }\n\n const initialVersion = toStateVersionString(lsn);\n\n initReplicationState(tx, publications, initialVersion, context);\n\n // Run up to MAX_WORKERS to copy of tables at the replication slot's snapshot.\n const start = performance.now();\n // Retrieve the published schema at the consistent_point.\n const published = await runTx(\n sql,\n async tx => {\n await tx.unsafe(/ sql/ `SET TRANSACTION SNAPSHOT '${snapshot}'`);\n return getPublicationInfo(tx, publications);\n },\n {mode: Mode.READONLY},\n );\n // Note: If this throws, initial-sync is aborted.\n validatePublications(lc, published);\n\n // Now that tables have been validated, kick off the copiers.\n const {tables, indexes} = published;\n const numTables = tables.length;\n if (platform() === 'win32' && tableCopyWorkers < numTables) {\n lc.warn?.(\n `Increasing the number of copy workers from ${tableCopyWorkers} to ` +\n `${numTables} to work around a Node/Postgres connection bug`,\n );\n }\n const numWorkers =\n platform() === 'win32'\n ? numTables\n : Math.min(tableCopyWorkers, numTables);\n\n const copyPool = await connectPgClient(\n lc,\n upstreamURI,\n 'initial-sync-copy-worker',\n {\n max: numWorkers,\n ['max_lifetime']: 120 60, // set a long (2h) limit for COPY streaming\n },\n );\n const copiers = startTableCopyWorkers(\n lc,\n copyPool,\n snapshot,\n numWorkers,\n numTables,\n );\n try {\n createLiteTables(tx, tables, initialVersion);\n const sampleRate = shadow?.sampleRate;\n const maxRowsPerTable = shadow?.maxRowsPerTable;\n const downloads = await Promise.all(\n tables.map(spec =>\n copiers.processReadTask((db, lc) =>\n getInitialDownloadState(lc, db, spec, shadow !== undefined),\n ),\n ),\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying ${numTables} upstream tables at version ${initialVersion}`,\n 5000,\n () => ({downloadStatus: downloads.map(({status}) => status)}),\n );\n\n void copyProfiler?.start();\n const rowCounts = await Promise.all(\n downloads.map(table =>\n copiers.processReadTask((db, lc) =>\n copy(\n lc,\n table,\n copyPool,\n db,\n tx,\n textCopy,\n sampleRate,\n maxRowsPerTable,\n ),\n ),\n ),\n );\n void copyProfiler?.stopAndDispose(lc, 'initial-copy');\n copiers.setDone();\n\n const total = rowCounts.reduce(\n (acc, curr) => ({\n rows: acc.rows + curr.rows,\n flushTime: acc.flushTime + curr.flushTime,\n }),\n {rows: 0, flushTime: 0},\n );\n\n statusPublisher.publish(\n lc,\n 'Indexing',\n `Creating ${indexes.length} indexes`,\n 5000,\n );\n const indexStart = performance.now();\n createLiteIndices(lc, tx, indexes);\n const index = performance.now() - indexStart;\n lc.info?.(`Created indexes (${index.toFixed(3)} ms)`);\n\n if (slotName && replicaID) {\n await initReplica(sql, shard, replicaID, published, context);\n } else {\n assert(shadow, 'expected to be in shadow sync if there is no slotName');\n const rowsByTable = new Map<string, number>();\n for (let i = 0; i < downloads.length; i++) {\n rowsByTable.set(downloads[i].status.table, rowCounts[i].rows);\n }\n verifyShadowReplica(lc, tx, published, rowsByTable);\n }\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Synced ${total.rows.toLocaleString()} rows of ${numTables} tables in ${publications} up to ${lsn} ` +\n `(flush: ${total.flushTime.toFixed(3)}, index: ${index.toFixed(3)}, total: ${elapsed.toFixed(3)} ms)`,\n );\n } finally {\n // All meaningful errors are handled at the processReadTask() call site.\n void copyPool.end().catch(e => lc.warn?.(`Error closing copyPool`, e));\n }\n } catch (e) {\n if (slotName) {\n // If initial-sync did not succeed, make a best effort to drop the\n // orphaned replication slot to avoid running out of slots in\n // pathological cases that result in repeated failures.\n lc.warn?.(`dropping replication slot ${slotName}`, e);\n await sql`\n SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n WHERE slot_name = ${slotName};\n `.catch(e => lc.warn?.(`Unable to drop replication slot ${slotName}`, e));\n }\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n if (releaseShadowSnapshot) {\n await releaseShadowSnapshot().catch(e =>\n lc.warn?.(`Error releasing shadow snapshot`, e),\n );\n }\n if (replicationSession) {\n await replicationSession.end();\n }\n await sql.end();\n }\n}\n\nexport type ShadowSyncOptions = {\n sampleRate: number;\n maxRowsPerTable: number;\n /*\n Parent directory for the throwaway SQLite replica. Defaults to the OS\n * tmpdir. Primarily for tests that need to isolate the scratch directory.\n /\n parentDir?: string \| undefined;\n};\n\n/\n Exercises the initial-sync code path against a sample of rows from every\n * published table, writing into a throwaway SQLite database that is deleted\n * when the run ends. Produces zero upstream mutations: no replication slot,\n * no `addReplica`, no `dropShard`, no status events.\n \n Intended to be invoked periodically so that if a customer ever needs a\n * full reset, we have recent confidence that `initialSync` still works.\n * The shard must already be initialized upstream.\n /\nexport async function shadowInitialSync(\n lc: LogContext,\n shard: ShardConfig,\n upstreamURI: string,\n shadow: ShadowSyncOptions,\n context: ServerContext,\n syncOptions?: Pick<InitialSyncOptions, 'textCopy'>,\n): Promise<void> {\n const dir = await mkdtemp(\n join(shadow.parentDir ?? tmpdir(), 'zero-shadow-sync-'),\n );\n const dbPath = join(dir, 'shadow-replica.db');\n const db = new Database(lc, dbPath);\n try {\n await initialSync(\n lc,\n shard,\n db,\n upstreamURI,\n {\n // Shadow sync copies small samples, so one worker is plenty —\n // no reason to burn additional upstream connections.\n tableCopyWorkers: 1,\n textCopy: syncOptions?.textCopy,\n shadow,\n },\n context,\n );\n } finally {\n try {\n db.close();\n } catch (e) {\n lc.warn?.(`Error closing shadow replica db`, e);\n }\n await rm(dir, {recursive: true, force: true}).catch(e =>\n lc.warn?.(`Error cleaning up shadow replica dir ${dir}`, e),\n );\n }\n}\n\nasync function checkUpstreamConfig(sql: PostgresDB) {\n const {walLevel, version} = (\n await sql<{walLevel: string; version: number}[]>`\n SELECT current_setting('wal_level') as \"walLevel\", \n current_setting('server_version_num') as \"version\";\n `\n )[0];\n\n if (walLevel !== 'logical') {\n throw new Error(\n `Postgres must be configured with \"wal_level = logical\" (currently: \"${walLevel})`,\n );\n }\n if (version < PG_15) {\n throw new Error(\n `Must be running Postgres 15 or higher (currently: \"${version}\")`,\n );\n }\n return version;\n}\n\nasync function ensurePublishedTables(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardConfig,\n validate = true,\n): Promise<{publications: string[]}> {\n const {database, host} = sql.options;\n lc.info?.(`Ensuring upstream PUBLICATION on ${database}@${host}`);\n\n await ensureShardSchema(lc, sql, shard);\n const {publications} = await getInternalShardConfig(sql, shard);\n\n if (validate) {\n let valid = false;\n const nonInternalPublications = publications.filter(\n p => !p.startsWith('_'),\n );\n const exists = await sql`\n SELECT pubname FROM pg_publication WHERE pubname IN ${sql(publications)}\n `.values();\n if (exists.length !== publications.length) {\n lc.warn?.(\n `some configured publications [${publications}] are missing: ` +\n `[${exists.flat()}]. resyncing`,\n );\n } else if (\n !equals(new Set(shard.publications), new Set(nonInternalPublications))\n ) {\n lc.warn?.(\n `requested publications [${shard.publications}] differ from previous` +\n `publications [${nonInternalPublications}]. resyncing`,\n );\n } else {\n valid = true;\n }\n if (!valid) {\n await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n return ensurePublishedTables(lc, sql, shard, false);\n }\n }\n return {publications};\n}\n\nfunction startTableCopyWorkers(\n lc: LogContext,\n db: PostgresDB,\n snapshot: string,\n numWorkers: number,\n numTables: number,\n): TransactionPool {\n const {init} = importSnapshot(snapshot);\n const tableCopiers = new TransactionPool(lc, {\n mode: Mode.READONLY,\n init,\n initialWorkers: numWorkers,\n });\n tableCopiers.run(db);\n\n lc.info?.(`Started ${numWorkers} workers to copy ${numTables} tables`);\n\n if (parseInt(process.versions.node) < 22) {\n lc.warn?.(\n `\\n\\n\\n` +\n `Older versions of Node have a bug that results in an unresponsive\\n` +\n `Postgres connection after running certain combinations of COPY commands.\\n` +\n `If initial sync hangs, run zero-cache with Node v22+. This has the additional\\n` +\n `benefit of being consistent with the Node version run in the production container image.` +\n `\\n\\n\\n`,\n );\n }\n return tableCopiers;\n}\n\n/\n Shadow-mode alternative to `createReplicationSlot`: opens a dedicated\n * READ ONLY REPEATABLE READ transaction on a normal connection, exports the\n * snapshot and captures the current WAL LSN, then holds the transaction\n * open until `release()` is called. The held transaction keeps the snapshot\n * importable by the table-copy workers for the duration of the COPY phase.\n \n Idle-in-transaction timeout is disabled locally so the exporter doesn't\n * get killed while workers are still importing.\n /\nasync function acquireExportedSnapshotForShadowSync(\n lc: LogContext,\n upstreamURI: string,\n): Promise<{\n snapshot: string;\n lsn: string;\n release: () => Promise<void>;\n}> {\n const holder = await connectPgClient(\n lc,\n upstreamURI,\n 'shadow-initial-sync-snapshot',\n {\n max: 1,\n },\n );\n const ready = resolver<{snapshot: string; lsn: string}>();\n const release = resolver<void>();\n const held = holder\n .begin(Mode.READONLY, async tx => {\n await tx`SET LOCAL idle_in_transaction_session_timeout = 0`.execute();\n const [row] = await tx<{snapshot: string; lsn: string}[]>`\n SELECT pg_export_snapshot() AS snapshot,\n pg_current_wal_lsn()::text AS lsn`;\n ready.resolve(row);\n await release.promise;\n })\n .catch(e => ready.reject(e));\n\n let snapshot: string;\n let lsn: string;\n try {\n ({snapshot, lsn} = await ready.promise);\n } catch (e) {\n await holder\n .end()\n .catch(err =>\n lc.warn?.(`Error ending shadow snapshot holder after failure`, err),\n );\n throw e;\n }\n lc.info?.(\n `Exported snapshot ${snapshot} at LSN ${lsn} (shadow initial sync)`,\n );\n return {\n snapshot,\n lsn,\n release: async () => {\n release.resolve();\n try {\n await held;\n } catch (e) {\n lc.warn?.(`snapshot holder transaction ended with error`, e);\n }\n await holder.end();\n },\n };\n}\n\nfunction createLiteTables(\n tx: Database,\n tables: PublishedTableSpec[],\n initialVersion: string,\n) {\n // TODO: Figure out how to reuse the ChangeProcessor here to avoid\n // duplicating the ColumnMetadata logic.\n const columnMetadata = must(ColumnMetadataStore.getInstance(tx));\n for (const t of tables) {\n tx.exec(createLiteTableStatement(mapPostgresToLite(t, initialVersion)));\n const tableName = liteTableName(t);\n for (const [colName, colSpec] of Object.entries(t.columns)) {\n columnMetadata.insert(tableName, colName, colSpec);\n }\n }\n}\n\nfunction createLiteIndices(lc: LogContext, tx: Database, indices: IndexSpec[]) {\n for (const [i, index] of indices.entries()) {\n const stmt = createLiteIndexStatement(mapPostgresToLiteIndex(index));\n lc.info?.(`Creating index ${i + 1}/${indices.length}: ${stmt}`);\n const start = performance.now();\n tx.exec(stmt);\n lc.info?.(\n `Created index ${i + 1}/${indices.length} ` +\n `(${(performance.now() - start).toFixed(3)} ms): ${stmt}`,\n );\n }\n}\n\n/\n Runs structural assertions over a just-synced replica and throws if any\n * fail. Only called in shadow mode — a successful return means the replica\n * is schema-complete, row-count consistent, and its column metadata is in\n * sync with its lite schema.\n \n Note: this intentionally does NOT verify ZQL-queryability. Tables that\n * `computeZqlSpecs` drops (no PK / no all-NOT-NULL unique index, unsupported\n * column types, etc.) are silently skipped in production too — there's\n * nothing shadow-specific about them, so failing here would diverge from\n * prod over an upstream-schema condition prod accepts.\n \n Exported for testing.\n /\nexport function verifyShadowReplica(\n lc: LogContext,\n db: Database,\n published: {tables: PublishedTableSpec[]; indexes: IndexSpec[]},\n rowsByTable: ReadonlyMap<string, number>,\n): void {\n const issues: string[] = [];\n let columnsChecked = 0;\n let rowsChecked = 0;\n\n // 1. Schema completeness: every published table exists in the replica\n // with at least the expected column set.\n const liteTables = listTables(db);\n const liteTableByName = new Map(liteTables.map(t => [t.name, t]));\n for (const pt of published.tables) {\n const name = liteTableName(pt);\n const lite = liteTableByName.get(name);\n if (!lite) {\n issues.push(`missing table in replica: ${name}`);\n continue;\n }\n for (const col of Object.keys(pt.columns)) {\n columnsChecked++;\n if (!(col in lite.columns)) {\n issues.push(`column missing in replica table ${name}: ${col}`);\n }\n }\n }\n\n // Every published index exists in the replica.\n const liteIndexNames = new Set(listIndexes(db).map(i => i.name));\n for (const ix of published.indexes) {\n const mapped = mapPostgresToLiteIndex(ix);\n if (!liteIndexNames.has(mapped.name)) {\n issues.push(\n `missing index in replica: ${mapped.name} on ${mapped.tableName}`,\n );\n }\n }\n\n // 2. Row counts: SQLite COUNT() matches the in-memory copy counter.\n for (const [table, expected] of rowsByTable) {\n try {\n const [row] = db\n .prepare(`SELECT COUNT() as count FROM \"${table}\"`)\n .all<{count: number}>();\n if (row.count !== expected) {\n issues.push(\n `row count mismatch for table ${table}: ` +\n `copy counter reported ${expected}, replica has ${row.count}`,\n );\n } else {\n rowsChecked += row.count;\n }\n } catch (e) {\n issues.push(`could not count rows in table ${table}: ${String(e)}`);\n }\n }\n\n // 3. Column metadata: every published column has a _zero.column_metadata row.\n const meta = must(ColumnMetadataStore.getInstance(db));\n for (const pt of published.tables) {\n const name = liteTableName(pt);\n const rows = meta.getTable(name);\n for (const col of Object.keys(pt.columns)) {\n if (!rows.has(col)) {\n issues.push(`missing column_metadata row for ${name}.${col}`);\n }\n }\n }\n\n if (issues.length) {\n throw new Error(\n `Shadow replica verification failed (${issues.length} issue(s)):\\n` +\n issues.map(i => ` - ${i}`).join('\\n'),\n );\n }\n\n lc.info?.(\n `Shadow replica verification passed: ` +\n `${published.tables.length} tables, ` +\n `${published.indexes.length} indexes, ` +\n `${columnsChecked} columns, ` +\n `${rowsChecked.toLocaleString()} rows`,\n );\n}\n\n// Verified empirically that batches of 50 seem to be the sweet spot,\n// similar to the report in https://sqlite.org/forum/forumpost/8878a512d3652655\n//\n// Exported for testing.\nexport const INSERT_BATCH_SIZE = 50;\n\nconst MB = 1024 1024;\nconst MAX_BUFFERED_ROWS = 10_000;\nconst BUFFERED_SIZE_THRESHOLD = 8 * MB;\n\nexport type DownloadStatements = {\n select: string;\n getTotalRows: string;\n getTotalBytes: string;\n};\n\n/*\n Produces ` TABLESAMPLE BERNOULLI(n)` when `sampleRate` is < 1, else `''`.\n * Row-level Bernoulli sampling is used (rather than SYSTEM) because it\n * produces a more uniform sample and, unlike SYSTEM, still returns rows\n * for small tables at low rates.\n /\nfunction tableSampleClause(sampleRate: number \| undefined): string {\n if (sampleRate === undefined \|\| sampleRate >= 1) {\n return '';\n }\n // Round away float noise (e.g. 0.3 100 = 30.000000000000004) while still\n // preserving sub-integer rates like 0.001 (= 0.1%).\n const pct = parseFloat((sampleRate * 100).toFixed(6));\n return /sql/ ` TABLESAMPLE BERNOULLI(${pct})`;\n}\n\nfunction limitClause(maxRowsPerTable: number \| undefined): string {\n return maxRowsPerTable !== undefined\n ? /sql/ ` LIMIT ${maxRowsPerTable}`\n : '';\n}\n\n/*\n Returns the SELECT column expressions for binary COPY, casting columns\n * without a known binary decoder to `::text`.\n /\nexport function makeBinarySelectExprs(\n table: PublishedTableSpec,\n cols: string[],\n): string[] {\n return cols.map(col => {\n const spec = table.columns[col];\n return hasBinaryDecoder(spec) ? id(col) : `${id(col)}::text`;\n });\n}\n\nexport function makeDownloadStatements(\n table: PublishedTableSpec,\n cols: string[],\n sampleRate?: number \| undefined,\n maxRowsPerTable?: number \| undefined,\n selectExprs?: string[] \| undefined,\n): DownloadStatements {\n const filterConditions = Object.values(table.publications)\n .map(({rowFilter}) => rowFilter)\n .filter(f => !!f); // remove nulls\n const where =\n filterConditions.length === 0\n ? ''\n : /sql/ `WHERE ${filterConditions.join(' OR ')}`;\n const sample = tableSampleClause(sampleRate);\n const limit = limitClause(maxRowsPerTable);\n const fromTable = /sql/ `FROM ${id(table.schema)}.${id(table.name)}${sample} ${where}`;\n const select = /sql/ `SELECT ${(selectExprs ?? cols.map(id)).join(',')} ${fromTable}${limit}`;\n if (limit) {\n // With LIMIT, wrap counts/sums in a subquery so they reflect the\n // capped rowset rather than the full (sampled) table.\n const bytesExpr = cols\n .map(col => `COALESCE(pg_column_size(${id(col)}), 0)`)\n .join(' + ');\n return {\n select,\n getTotalRows: /sql/ `SELECT COUNT()::bigint AS \"totalRows\" FROM (SELECT 1 AS _ ${fromTable}${limit}) s`,\n getTotalBytes: /sql/ `SELECT COALESCE(SUM(b), 0)::bigint AS \"totalBytes\" FROM (SELECT (${bytesExpr}) AS b ${fromTable}${limit}) s`,\n };\n }\n const totalBytes = `(${cols.map(col => `SUM(COALESCE(pg_column_size(${id(col)}), 0))`).join(' + ')})`;\n return {\n select,\n getTotalRows: /sql/ `SELECT COUNT() AS \"totalRows\" ${fromTable}`,\n getTotalBytes: /sql/ `SELECT ${totalBytes} AS \"totalBytes\" ${fromTable}`,\n };\n}\n\ntype DownloadState = {\n spec: PublishedTableSpec;\n status: DownloadStatus;\n};\n\n// Exported for testing.\nexport async function getInitialDownloadState(\n lc: LogContext,\n sql: PostgresDB,\n spec: PublishedTableSpec,\n skipTotals: boolean,\n): Promise<DownloadState> {\n const start = performance.now();\n const table = liteTableName(spec);\n const columns = Object.keys(spec.columns);\n if (skipTotals) {\n // Shadow sync suppresses status events, so the pg_class\n // estimates would be computed and thrown away.\n return {\n spec,\n status: {table, columns, rows: 0, totalRows: 0, totalBytes: 0},\n };\n }\n // Use pg_class estimates instead of expensive COUNT() and\n // SUM(pg_column_size(...)) full table scans. The exact values are only\n // used for progress reporting, so estimates are sufficient.\n const qualifiedName = `${id(spec.schema)}.${id(spec.name)}`;\n const estimateResult = await sql<\n {totalRows: number; totalBytes: number}[]\n >`SELECT GREATEST(reltuples, 0)::float8 AS \"totalRows\",\n pg_table_size(oid)::float8 AS \"totalBytes\"\n FROM pg_class\n WHERE oid = ${qualifiedName}::regclass`;\n\n const {totalRows, totalBytes} = estimateResult[0] ?? {\n totalRows: 0,\n totalBytes: 0,\n };\n\n const state: DownloadState = {\n spec,\n status: {\n table,\n columns,\n rows: 0,\n totalRows,\n totalBytes,\n },\n };\n const elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(`Computed initial download state for ${table} (${elapsed} ms)`, {\n state: state.status,\n });\n return state;\n}\n\nfunction copy(\n lc: LogContext,\n {spec: table, status}: DownloadState,\n dbClient: PostgresDB,\n from: PostgresTransaction,\n to: Database,\n textCopy: boolean,\n sampleRate?: number \| undefined,\n maxRowsPerTable?: number \| undefined,\n) {\n if (textCopy) {\n return copyText(\n lc,\n table,\n status,\n dbClient,\n from,\n to,\n sampleRate,\n maxRowsPerTable,\n );\n }\n return copyBinary(lc, table, status, from, to, sampleRate, maxRowsPerTable);\n}\n\nasync function copyBinary(\n lc: LogContext,\n table: PublishedTableSpec,\n status: DownloadStatus,\n from: PostgresTransaction,\n to: Database,\n sampleRate?: number \| undefined,\n maxRowsPerTable?: number \| undefined,\n) {\n const start = performance.now();\n let flushTime = 0;\n\n const tableName = liteTableName(table);\n const orderedColumns = Object.entries(table.columns);\n\n const columnNames = orderedColumns.map(([c]) => c);\n const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n const valuesSql =\n columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n const insertSql = /sql/ `\n INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n const insertStmt = to.prepare(insertSql);\n const insertBatchStmt = to.prepare(\n insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n );\n\n // Build SELECT with ::text casts for columns without a known binary decoder.\n const select = makeDownloadStatements(\n table,\n columnNames,\n sampleRate,\n maxRowsPerTable,\n makeBinarySelectExprs(table, columnNames),\n ).select;\n\n const decoders = orderedColumns.map(([, spec]) =>\n hasBinaryDecoder(spec) ? makeBinaryDecoder(spec) : textCastDecoder,\n );\n\n const valuesPerRow = columnSpecs.length;\n const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;\n\n const pendingValues: LiteValueType[] = Array.from({\n length: MAX_BUFFERED_ROWS * valuesPerRow,\n });\n let pendingRows = 0;\n let pendingSize = 0;\n\n function flush() {\n const start = performance.now();\n const flushedRows = pendingRows;\n const flushedSize = pendingSize;\n\n let l = 0;\n for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n }\n for (; pendingRows > 0; pendingRows--) {\n insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n }\n const flushedValues = flushedRows * valuesPerRow;\n for (let i = 0; i < flushedValues; i++) {\n pendingValues[i] = undefined as unknown as LiteValueType;\n }\n pendingSize = 0;\n status.rows += flushedRows;\n\n const elapsed = performance.now() - start;\n flushTime += elapsed;\n lc.debug?.(\n `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n );\n }\n\n const binaryParser = new BinaryCopyParser();\n let col = 0;\n\n lc.info?.(`Starting binary copy stream of ${tableName}:`, select);\n\n await pipeline(\n await from\n .unsafe(`COPY (${select}) TO STDOUT WITH (FORMAT binary)`)\n .readable(),\n new Writable({\n highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n write(\n chunk: Buffer,\n _encoding: string,\n callback: (error?: Error) => void,\n ) {\n try {\n for (const fieldBuf of binaryParser.parse(chunk)) {\n pendingSize += fieldBuf === null ? 4 : fieldBuf.length;\n pendingValues[pendingRows * valuesPerRow + col] =\n fieldBuf === null ? null : decoders[col](fieldBuf);\n\n if (++col === decoders.length) {\n col = 0;\n if (\n ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow \|\|\n pendingSize >= BUFFERED_SIZE_THRESHOLD\n ) {\n flush();\n }\n }\n }\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n\n final: (callback: (error?: Error) => void) => {\n try {\n flush();\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n }),\n );\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Finished copying ${status.rows} rows into ${tableName} ` +\n `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n );\n return {rows: status.rows, flushTime};\n}\n\nasync function copyText(\n lc: LogContext,\n table: PublishedTableSpec,\n status: DownloadStatus,\n dbClient: PostgresDB,\n from: PostgresTransaction,\n to: Database,\n sampleRate?: number \| undefined,\n maxRowsPerTable?: number \| undefined,\n) {\n const start = performance.now();\n let flushTime = 0;\n\n const tableName = liteTableName(table);\n const orderedColumns = Object.entries(table.columns);\n\n const columnNames = orderedColumns.map(([c]) => c);\n const columnSpecs = orderedColumns.map(([_name, spec]) => spec);\n const insertColumnList = columnNames.map(c => id(c)).join(',');\n\n const valuesSql =\n columnNames.length > 0 ? `(${'?,'.repeat(columnNames.length - 1)}?)` : '()';\n const insertSql = /sql/ `\n INSERT INTO \"${tableName}\" (${insertColumnList}) VALUES ${valuesSql}`;\n const insertStmt = to.prepare(insertSql);\n const insertBatchStmt = to.prepare(\n insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1),\n );\n\n const {select} = makeDownloadStatements(\n table,\n columnNames,\n sampleRate,\n maxRowsPerTable,\n );\n const valuesPerRow = columnSpecs.length;\n const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;\n\n const pendingValues: LiteValueType[] = Array.from({\n length: MAX_BUFFERED_ROWS * valuesPerRow,\n });\n let pendingRows = 0;\n let pendingSize = 0;\n\n function flush() {\n const start = performance.now();\n const flushedRows = pendingRows;\n const flushedSize = pendingSize;\n\n let l = 0;\n for (; pendingRows > INSERT_BATCH_SIZE; pendingRows -= INSERT_BATCH_SIZE) {\n insertBatchStmt.run(pendingValues.slice(l, (l += valuesPerBatch)));\n }\n for (; pendingRows > 0; pendingRows--) {\n insertStmt.run(pendingValues.slice(l, (l += valuesPerRow)));\n }\n const flushedValues = flushedRows * valuesPerRow;\n for (let i = 0; i < flushedValues; i++) {\n pendingValues[i] = undefined as unknown as LiteValueType;\n }\n pendingSize = 0;\n status.rows += flushedRows;\n\n const elapsed = performance.now() - start;\n flushTime += elapsed;\n lc.debug?.(\n `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed.toFixed(3)} ms`,\n );\n }\n\n lc.info?.(`Starting text copy stream of ${tableName}:`, select);\n const pgParsers = await getTypeParsers(dbClient, {returnJsonAsString: true});\n const parsers = columnSpecs.map(c => {\n const pgParse = pgParsers.getTypeParser(c.typeOID);\n return (val: string) =>\n liteValue(\n pgParse(val) as PostgresValueType,\n c.dataType,\n JSON_STRINGIFIED,\n );\n });\n\n const tsvParser = new TsvParser();\n let col = 0;\n\n await pipeline(\n await from.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n new Writable({\n highWaterMark: BUFFERED_SIZE_THRESHOLD,\n\n write(\n chunk: Buffer,\n _encoding: string,\n callback: (error?: Error) => void,\n ) {\n try {\n for (const text of tsvParser.parse(chunk)) {\n pendingSize += text === null ? 4 : text.length;\n pendingValues[pendingRows * valuesPerRow + col] =\n text === null ? null : parsers[col](text);\n\n if (++col === parsers.length) {\n col = 0;\n if (\n ++pendingRows >= MAX_BUFFERED_ROWS - valuesPerRow \|\|\n pendingSize >= BUFFERED_SIZE_THRESHOLD\n ) {\n flush();\n }\n }\n }\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n\n final: (callback: (error?: Error) => void) => {\n try {\n flush();\n callback();\n } catch (e) {\n callback(e instanceof Error ? e : new Error(String(e)));\n }\n },\n }),\n );\n\n const elapsed = performance.now() - start;\n lc.info?.(\n `Finished copying ${status.rows} rows into ${tableName} ` +\n `(flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `,\n );\n return {rows: status.rows, flushTime};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8FA,eAAsB,YACpB,IACA,OACA,IACA,aACA,aACA,SACA;AACA,KAAI,CAAC,0BAA0B,KAAK,MAAM,MAAM,CAC9C,OAAM,IAAI,MACR,2FACD;CAEH,MAAM,EACJ,kBACA,aACA,WAAW,OACX,0BAA0B,OAC1B,WACE;CACJ,MAAM,eAAe,cAAc,MAAM,YAAY,SAAS,GAAG;CACjE,MAAM,MAAM,MAAM,gBAAgB,IAAI,aAAa,eAAe;CAIlE,MAAM,qBAAqB,SACvB,KAAA,IACA,SAAS,IAAI,aAAa,oCAAoC;GAC3D,gBAAgB;EACjB,YAAY,EAAC,aAAa,YAAW;EACtC,CAAC;CAEN,MAAM,YAAY,KAAK,KAAK,CAAC,UAAU;CACvC,IAAI;CACJ,MAAM,kBAAkB,2BAA2B,sBACjD,IACA,SAAS,YAAY,KAAK,KAAA,EAC3B,CAAC,QAAQ,IAAI,eAAe;CAC7B,IAAI;AACJ,KAAI;EACF,MAAM,YAAY,MAAM,oBAAoB,IAAI;EAMhD,MAAM,EAAC,iBAAgB,SACnB,MAAM,uBAAuB,KAAK,MAAM,GACxC,MAAM,sBAAsB,IAAI,KAAK,MAAM;AAC/C,KAAG,OAAO,wCAAwC,aAAa,GAAG;EAElE,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,KAAG,OACD,SACI,kCAAkC,SAAS,GAAG,KAAK,kBACnD,kCAAkC,SAAS,GAAG,OACnD;EAED,IAAI;EACJ,IAAI;AAEJ,MAAI,QAAQ;GACV,MAAM,WAAW,MAAM,qCACrB,IACA,YACD;AACD,cAAW,SAAS;AACpB,SAAM,SAAS;AACf,2BAAwB,SAAS;SAC5B;GACL,MAAM,OAAO,MAAM,qBACjB,IACA,KACA,KAAK,mBAAmB,EACxB,OACA,WACA,2BAA2B,aAAA,KAC5B;AACD,cAAW,KAAK;AAChB,SAAM,KAAK;AACX,cAAW,KAAK;;EAGlB,MAAM,iBAAiB,qBAAqB,IAAI;AAEhD,uBAAqB,IAAI,cAAc,gBAAgB,QAAQ;EAG/D,MAAM,QAAQ,YAAY,KAAK;EAE/B,MAAM,YAAY,MAAM,MACtB,KACA,OAAM,OAAM;AACV,SAAM,GAAG,OAAgB,6BAA6B,SAAS,GAAG;AAClE,UAAO,mBAAmB,IAAI,aAAa;KAE7C,EAAC,MAAM,UAAc,CACtB;AAED,uBAAqB,IAAI,UAAU;EAGnC,MAAM,EAAC,QAAQ,YAAW;EAC1B,MAAM,YAAY,OAAO;AACzB,MAAI,UAAU,KAAK,WAAW,mBAAmB,UAC/C,IAAG,OACD,8CAA8C,iBAAiB,MAC1D,UAAU,gDAChB;EAEH,MAAM,aACJ,UAAU,KAAK,UACX,YACA,KAAK,IAAI,kBAAkB,UAAU;EAE3C,MAAM,WAAW,MAAM,gBACrB,IACA,aACA,4BACA;GACE,KAAK;IACJ,iBAAiB;GACnB,CACF;EACD,MAAM,UAAU,sBACd,IACA,UACA,UACA,YACA,UACD;AACD,MAAI;AACF,oBAAiB,IAAI,QAAQ,eAAe;GAC5C,MAAM,aAAa,QAAQ;GAC3B,MAAM,kBAAkB,QAAQ;GAChC,MAAM,YAAY,MAAM,QAAQ,IAC9B,OAAO,KAAI,SACT,QAAQ,iBAAiB,IAAI,OAC3B,wBAAwB,IAAI,IAAI,MAAM,WAAW,KAAA,EAAU,CAC5D,CACF,CACF;AACD,mBAAgB,QACd,IACA,gBACA,WAAW,UAAU,8BAA8B,kBACnD,YACO,EAAC,gBAAgB,UAAU,KAAK,EAAC,aAAY,OAAO,EAAC,EAC7D;AAEI,iBAAc,OAAO;GAC1B,MAAM,YAAY,MAAM,QAAQ,IAC9B,UAAU,KAAI,UACZ,QAAQ,iBAAiB,IAAI,OAC3B,KACE,IACA,OACA,UACA,IACA,IACA,UACA,YACA,gBACD,CACF,CACF,CACF;AACI,iBAAc,eAAe,IAAI,eAAe;AACrD,WAAQ,SAAS;GAEjB,MAAM,QAAQ,UAAU,QACrB,KAAK,UAAU;IACd,MAAM,IAAI,OAAO,KAAK;IACtB,WAAW,IAAI,YAAY,KAAK;IACjC,GACD;IAAC,MAAM;IAAG,WAAW;IAAE,CACxB;AAED,mBAAgB,QACd,IACA,YACA,YAAY,QAAQ,OAAO,WAC3B,IACD;GACD,MAAM,aAAa,YAAY,KAAK;AACpC,qBAAkB,IAAI,IAAI,QAAQ;GAClC,MAAM,QAAQ,YAAY,KAAK,GAAG;AAClC,MAAG,OAAO,oBAAoB,MAAM,QAAQ,EAAE,CAAC,MAAM;AAErD,OAAI,YAAY,UACd,OAAM,YAAY,KAAK,OAAO,WAAW,WAAW,QAAQ;QACvD;AACL,WAAO,QAAQ,wDAAwD;IACvE,MAAM,8BAAc,IAAI,KAAqB;AAC7C,SAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,IACpC,aAAY,IAAI,UAAU,GAAG,OAAO,OAAO,UAAU,GAAG,KAAK;AAE/D,wBAAoB,IAAI,IAAI,WAAW,YAAY;;GAGrD,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,MAAG,OACD,UAAU,MAAM,KAAK,gBAAgB,CAAC,WAAW,UAAU,aAAa,aAAa,SAAS,IAAI,WACrF,MAAM,UAAU,QAAQ,EAAE,CAAC,WAAW,MAAM,QAAQ,EAAE,CAAC,WAAW,QAAQ,QAAQ,EAAE,CAAC,MACnG;YACO;AAEH,YAAS,KAAK,CAAC,OAAM,MAAK,GAAG,OAAO,0BAA0B,EAAE,CAAC;;UAEjE,GAAG;AACV,MAAI,UAAU;AAIZ,MAAG,OAAO,6BAA6B,YAAY,EAAE;AACrD,SAAM,GAAG;;8BAEe,SAAS;QAC/B,OAAM,MAAK,GAAG,OAAO,mCAAmC,YAAY,EAAE,CAAC;;AAE3E,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;AACtB,MAAI,sBACF,OAAM,uBAAuB,CAAC,OAAM,MAClC,GAAG,OAAO,mCAAmC,EAAE,CAChD;AAEH,MAAI,mBACF,OAAM,mBAAmB,KAAK;AAEhC,QAAM,IAAI,KAAK;;;;;;;;;;;;;AAwBnB,eAAsB,kBACpB,IACA,OACA,aACA,QACA,SACA,aACe;CACf,MAAM,MAAM,MAAM,QAChB,KAAK,OAAO,aAAa,QAAQ,EAAE,oBAAoB,CACxD;CAED,MAAM,KAAK,IAAI,SAAS,IADT,KAAK,KAAK,oBAAoB,CACV;AACnC,KAAI;AACF,QAAM,YACJ,IACA,OACA,IACA,aACA;GAGE,kBAAkB;GAClB,UAAU,aAAa;GACvB;GACD,EACD,QACD;WACO;AACR,MAAI;AACF,MAAG,OAAO;WACH,GAAG;AACV,MAAG,OAAO,mCAAmC,EAAE;;AAEjD,QAAM,GAAG,KAAK;GAAC,WAAW;GAAM,OAAO;GAAK,CAAC,CAAC,OAAM,MAClD,GAAG,OAAO,wCAAwC,OAAO,EAAE,CAC5D;;;AAIL,eAAe,oBAAoB,KAAiB;CAClD,MAAM,EAAC,UAAU,aACf,MAAM,GAA0C;;;KAIhD;AAEF,KAAI,aAAa,UACf,OAAM,IAAI,MACR,uEAAuE,SAAS,GACjF;AAEH,KAAI,UAAA,KACF,OAAM,IAAI,MACR,sDAAsD,QAAQ,IAC/D;AAEH,QAAO;;AAGT,eAAe,sBACb,IACA,KACA,OACA,WAAW,MACwB;CACnC,MAAM,EAAC,UAAU,SAAQ,IAAI;AAC7B,IAAG,OAAO,oCAAoC,SAAS,GAAG,OAAO;AAEjE,OAAM,kBAAkB,IAAI,KAAK,MAAM;CACvC,MAAM,EAAC,iBAAgB,MAAM,uBAAuB,KAAK,MAAM;AAE/D,KAAI,UAAU;EACZ,IAAI,QAAQ;EACZ,MAAM,0BAA0B,aAAa,QAC3C,MAAK,CAAC,EAAE,WAAW,IAAI,CACxB;EACD,MAAM,SAAS,MAAM,GAAG;4DACgC,IAAI,aAAa,CAAC;QACtE,QAAQ;AACZ,MAAI,OAAO,WAAW,aAAa,OACjC,IAAG,OACD,iCAAiC,aAAa,kBACxC,OAAO,MAAM,CAAC,cACrB;WAED,CAAC,OAAO,IAAI,IAAI,MAAM,aAAa,EAAE,IAAI,IAAI,wBAAwB,CAAC,CAEtE,IAAG,OACD,2BAA2B,MAAM,aAAa,sCAC3B,wBAAwB,cAC5C;MAED,SAAQ;AAEV,MAAI,CAAC,OAAO;AACV,SAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,SAAS,CAAC;AACxD,UAAO,sBAAsB,IAAI,KAAK,OAAO,MAAM;;;AAGvD,QAAO,EAAC,cAAa;;AAGvB,SAAS,sBACP,IACA,IACA,UACA,YACA,WACiB;CACjB,MAAM,EAAC,SAAQ,eAAe,SAAS;CACvC,MAAM,eAAe,IAAI,gBAAgB,IAAI;EAC3C,MAAM;EACN;EACA,gBAAgB;EACjB,CAAC;AACF,cAAa,IAAI,GAAG;AAEpB,IAAG,OAAO,WAAW,WAAW,mBAAmB,UAAU,SAAS;AAEtE,KAAI,SAAS,QAAQ,SAAS,KAAK,GAAG,GACpC,IAAG,OACD,mUAMD;AAEH,QAAO;;;;;;;;;;;;AAaT,eAAe,qCACb,IACA,aAKC;CACD,MAAM,SAAS,MAAM,gBACnB,IACA,aACA,gCACA,EACE,KAAK,GACN,CACF;CACD,MAAM,QAAQ,UAA2C;CACzD,MAAM,UAAU,UAAgB;CAChC,MAAM,OAAO,OACV,MAAM,UAAe,OAAM,OAAM;AAChC,QAAM,EAAE,oDAAoD,SAAS;EACrE,MAAM,CAAC,OAAO,MAAM,EAAqC;;;AAGzD,QAAM,QAAQ,IAAI;AAClB,QAAM,QAAQ;GACd,CACD,OAAM,MAAK,MAAM,OAAO,EAAE,CAAC;CAE9B,IAAI;CACJ,IAAI;AACJ,KAAI;AACF,GAAC,CAAC,UAAU,OAAO,MAAM,MAAM;UACxB,GAAG;AACV,QAAM,OACH,KAAK,CACL,OAAM,QACL,GAAG,OAAO,qDAAqD,IAAI,CACpE;AACH,QAAM;;AAER,IAAG,OACD,qBAAqB,SAAS,UAAU,IAAI,wBAC7C;AACD,QAAO;EACL;EACA;EACA,SAAS,YAAY;AACnB,WAAQ,SAAS;AACjB,OAAI;AACF,UAAM;YACC,GAAG;AACV,OAAG,OAAO,gDAAgD,EAAE;;AAE9D,SAAM,OAAO,KAAK;;EAErB;;AAGH,SAAS,iBACP,IACA,QACA,gBACA;CAGA,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,GAAG,CAAC;AAChE,MAAK,MAAM,KAAK,QAAQ;AACtB,KAAG,KAAK,yBAAyB,kBAAkB,GAAG,eAAe,CAAC,CAAC;EACvE,MAAM,YAAY,cAAc,EAAE;AAClC,OAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,EAAE,QAAQ,CACxD,gBAAe,OAAO,WAAW,SAAS,QAAQ;;;AAKxD,SAAS,kBAAkB,IAAgB,IAAc,SAAsB;AAC7E,MAAK,MAAM,CAAC,GAAG,UAAU,QAAQ,SAAS,EAAE;EAC1C,MAAM,OAAO,yBAAyB,uBAAuB,MAAM,CAAC;AACpE,KAAG,OAAO,kBAAkB,IAAI,EAAE,GAAG,QAAQ,OAAO,IAAI,OAAO;EAC/D,MAAM,QAAQ,YAAY,KAAK;AAC/B,KAAG,KAAK,KAAK;AACb,KAAG,OACD,iBAAiB,IAAI,EAAE,GAAG,QAAQ,OAAO,KAClC,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE,CAAC,QAAQ,OACtD;;;;;;;;;;;;;;;;;AAkBL,SAAgB,oBACd,IACA,IACA,WACA,aACM;CACN,MAAM,SAAmB,EAAE;CAC3B,IAAI,iBAAiB;CACrB,IAAI,cAAc;CAIlB,MAAM,aAAa,WAAW,GAAG;CACjC,MAAM,kBAAkB,IAAI,IAAI,WAAW,KAAI,MAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;AACjE,MAAK,MAAM,MAAM,UAAU,QAAQ;EACjC,MAAM,OAAO,cAAc,GAAG;EAC9B,MAAM,OAAO,gBAAgB,IAAI,KAAK;AACtC,MAAI,CAAC,MAAM;AACT,UAAO,KAAK,6BAA6B,OAAO;AAChD;;AAEF,OAAK,MAAM,OAAO,OAAO,KAAK,GAAG,QAAQ,EAAE;AACzC;AACA,OAAI,EAAE,OAAO,KAAK,SAChB,QAAO,KAAK,mCAAmC,KAAK,IAAI,MAAM;;;CAMpE,MAAM,iBAAiB,IAAI,IAAI,YAAY,GAAG,CAAC,KAAI,MAAK,EAAE,KAAK,CAAC;AAChE,MAAK,MAAM,MAAM,UAAU,SAAS;EAClC,MAAM,SAAS,uBAAuB,GAAG;AACzC,MAAI,CAAC,eAAe,IAAI,OAAO,KAAK,CAClC,QAAO,KACL,6BAA6B,OAAO,KAAK,MAAM,OAAO,YACvD;;AAKL,MAAK,MAAM,CAAC,OAAO,aAAa,YAC9B,KAAI;EACF,MAAM,CAAC,OAAO,GACX,QAAQ,kCAAkC,MAAM,GAAG,CACnD,KAAsB;AACzB,MAAI,IAAI,UAAU,SAChB,QAAO,KACL,gCAAgC,MAAM,0BACX,SAAS,gBAAgB,IAAI,QACzD;MAED,gBAAe,IAAI;UAEd,GAAG;AACV,SAAO,KAAK,iCAAiC,MAAM,IAAI,OAAO,EAAE,GAAG;;CAKvE,MAAM,OAAO,KAAK,oBAAoB,YAAY,GAAG,CAAC;AACtD,MAAK,MAAM,MAAM,UAAU,QAAQ;EACjC,MAAM,OAAO,cAAc,GAAG;EAC9B,MAAM,OAAO,KAAK,SAAS,KAAK;AAChC,OAAK,MAAM,OAAO,OAAO,KAAK,GAAG,QAAQ,CACvC,KAAI,CAAC,KAAK,IAAI,IAAI,CAChB,QAAO,KAAK,mCAAmC,KAAK,GAAG,MAAM;;AAKnE,KAAI,OAAO,OACT,OAAM,IAAI,MACR,uCAAuC,OAAO,OAAO,iBACnD,OAAO,KAAI,MAAK,OAAO,IAAI,CAAC,KAAK,KAAK,CACzC;AAGH,IAAG,OACD,uCACK,UAAU,OAAO,OAAO,WACxB,UAAU,QAAQ,OAAO,YACzB,eAAe,YACf,YAAY,gBAAgB,CAAC,OACnC;;AASH,IAAM,KAAK,OAAO;AAClB,IAAM,oBAAoB;AAC1B,IAAM,0BAA0B,IAAI;;;;;;;AAcpC,SAAS,kBAAkB,YAAwC;AACjE,KAAI,eAAe,KAAA,KAAa,cAAc,EAC5C,QAAO;AAKT,QAAe,0BADH,YAAY,aAAa,KAAK,QAAQ,EAAE,CAAC,CACR;;AAG/C,SAAS,YAAY,iBAA6C;AAChE,QAAO,oBAAoB,KAAA,IACf,UAAU,oBAClB;;;;;;AAON,SAAgB,sBACd,OACA,MACU;AACV,QAAO,KAAK,KAAI,QAAO;EACrB,MAAM,OAAO,MAAM,QAAQ;AAC3B,SAAO,iBAAiB,KAAK,GAAG,GAAG,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC;GACrD;;AAGJ,SAAgB,uBACd,OACA,MACA,YACA,iBACA,aACoB;CACpB,MAAM,mBAAmB,OAAO,OAAO,MAAM,aAAa,CACvD,KAAK,EAAC,gBAAe,UAAU,CAC/B,QAAO,MAAK,CAAC,CAAC,EAAE;CACnB,MAAM,QACJ,iBAAiB,WAAW,IACxB,KACQ,SAAS,iBAAiB,KAAK,OAAO;CACpD,MAAM,SAAS,kBAAkB,WAAW;CAC5C,MAAM,QAAQ,YAAY,gBAAgB;CAC1C,MAAM,YAAoB,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,GAAG,OAAO,GAAG;CACjF,MAAM,SAAiB,WAAW,eAAe,KAAK,IAAI,GAAG,EAAE,KAAK,IAAI,CAAC,GAAG,YAAY;AACxF,KAAI,OAAO;EAGT,MAAM,YAAY,KACf,KAAI,QAAO,2BAA2B,GAAG,IAAI,CAAC,OAAO,CACrD,KAAK,MAAM;AACd,SAAO;GACL;GACA,cAAsB,8DAA8D,YAAY,MAAM;GACtG,eAAuB,oEAAoE,UAAU,SAAS,YAAY,MAAM;GACjI;;CAEH,MAAM,aAAa,IAAI,KAAK,KAAI,QAAO,+BAA+B,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;AACnG,QAAO;EACL;EACA,cAAsB,kCAAkC;EACxD,eAAuB,UAAU,WAAW,mBAAmB;EAChE;;AASH,eAAsB,wBACpB,IACA,KACA,MACA,YACwB;CACxB,MAAM,QAAQ,YAAY,KAAK;CAC/B,MAAM,QAAQ,cAAc,KAAK;CACjC,MAAM,UAAU,OAAO,KAAK,KAAK,QAAQ;AACzC,KAAI,WAGF,QAAO;EACL;EACA,QAAQ;GAAC;GAAO;GAAS,MAAM;GAAG,WAAW;GAAG,YAAY;GAAE;EAC/D;CAaH,MAAM,EAAC,WAAW,gBAPK,MAAM,GAE5B;;;kBAHqB,GAAG,GAAG,KAAK,OAAO,CAAC,GAAG,GAAG,KAAK,KAAK,GAM3B,aAEiB,MAAM;EACnD,WAAW;EACX,YAAY;EACb;CAED,MAAM,QAAuB;EAC3B;EACA,QAAQ;GACN;GACA;GACA,MAAM;GACN;GACA;GACD;EACF;CACD,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,IAAG,OAAO,uCAAuC,MAAM,IAAI,QAAQ,OAAO,EACxE,OAAO,MAAM,QACd,CAAC;AACF,QAAO;;AAGT,SAAS,KACP,IACA,EAAC,MAAM,OAAO,UACd,UACA,MACA,IACA,UACA,YACA,iBACA;AACA,KAAI,SACF,QAAO,SACL,IACA,OACA,QACA,UACA,MACA,IACA,YACA,gBACD;AAEH,QAAO,WAAW,IAAI,OAAO,QAAQ,MAAM,IAAI,YAAY,gBAAgB;;AAG7E,eAAe,WACb,IACA,OACA,QACA,MACA,IACA,YACA,iBACA;CACA,MAAM,QAAQ,YAAY,KAAK;CAC/B,IAAI,YAAY;CAEhB,MAAM,YAAY,cAAc,MAAM;CACtC,MAAM,iBAAiB,OAAO,QAAQ,MAAM,QAAQ;CAEpD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,EAAE;CAClD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,UAAU,KAAK;CAC/D,MAAM,mBAAmB,YAAY,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI;CAE9D,MAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,EAAE,CAAC,MAAM;CACzE,MAAM,YAAoB;mBACT,UAAU,KAAK,iBAAiB,WAAW;CAC5D,MAAM,aAAa,GAAG,QAAQ,UAAU;CACxC,MAAM,kBAAkB,GAAG,QACzB,YAAY,IAAI,YAAY,OAAA,GAA6B,CAC1D;CAGD,MAAM,SAAS,uBACb,OACA,aACA,YACA,iBACA,sBAAsB,OAAO,YAAY,CAC1C,CAAC;CAEF,MAAM,WAAW,eAAe,KAAK,GAAG,UACtC,iBAAiB,KAAK,GAAG,kBAAkB,KAAK,GAAG,gBACpD;CAED,MAAM,eAAe,YAAY;CACjC,MAAM,iBAAiB,eAAA;CAEvB,MAAM,gBAAiC,MAAM,KAAK,EAChD,QAAQ,oBAAoB,cAC7B,CAAC;CACF,IAAI,cAAc;CAClB,IAAI,cAAc;CAElB,SAAS,QAAQ;EACf,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,cAAc;EACpB,MAAM,cAAc;EAEpB,IAAI,IAAI;AACR,SAAO,cAAA,IAAiC,eAAA,GACtC,iBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,eAAgB,CAAC;AAEpE,SAAO,cAAc,GAAG,cACtB,YAAW,IAAI,cAAc,MAAM,GAAI,KAAK,aAAc,CAAC;EAE7D,MAAM,gBAAgB,cAAc;AACpC,OAAK,IAAI,IAAI,GAAG,IAAI,eAAe,IACjC,eAAc,KAAK,KAAA;AAErB,gBAAc;AACd,SAAO,QAAQ;EAEf,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,eAAa;AACb,KAAG,QACD,WAAW,YAAY,GAAG,UAAU,SAAS,YAAY,aAAa,QAAQ,QAAQ,EAAE,CAAC,KAC1F;;CAGH,MAAM,eAAe,IAAI,kBAAkB;CAC3C,IAAI,MAAM;AAEV,IAAG,OAAO,kCAAkC,UAAU,IAAI,OAAO;AAEjE,OAAM,WACJ,MAAM,KACH,OAAO,SAAS,OAAO,kCAAkC,CACzD,UAAU,EACb,IAAI,SAAS;EACX,eAAe;EAEf,MACE,OACA,WACA,UACA;AACA,OAAI;AACF,SAAK,MAAM,YAAY,aAAa,MAAM,MAAM,EAAE;AAChD,oBAAe,aAAa,OAAO,IAAI,SAAS;AAChD,mBAAc,cAAc,eAAe,OACzC,aAAa,OAAO,OAAO,SAAS,KAAK,SAAS;AAEpD,SAAI,EAAE,QAAQ,SAAS,QAAQ;AAC7B,YAAM;AACN,UACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,wBAEf,QAAO;;;AAIb,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAI3D,QAAQ,aAAsC;AAC5C,OAAI;AACF,WAAO;AACP,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAG5D,CAAC,CACH;CAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,IAAG,OACD,oBAAoB,OAAO,KAAK,aAAa,UAAU,WAC1C,UAAU,QAAQ,EAAE,CAAC,eAAe,QAAQ,QAAQ,EAAE,CAAC,OACrE;AACD,QAAO;EAAC,MAAM,OAAO;EAAM;EAAU;;AAGvC,eAAe,SACb,IACA,OACA,QACA,UACA,MACA,IACA,YACA,iBACA;CACA,MAAM,QAAQ,YAAY,KAAK;CAC/B,IAAI,YAAY;CAEhB,MAAM,YAAY,cAAc,MAAM;CACtC,MAAM,iBAAiB,OAAO,QAAQ,MAAM,QAAQ;CAEpD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,EAAE;CAClD,MAAM,cAAc,eAAe,KAAK,CAAC,OAAO,UAAU,KAAK;CAC/D,MAAM,mBAAmB,YAAY,KAAI,MAAK,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI;CAE9D,MAAM,YACJ,YAAY,SAAS,IAAI,IAAI,KAAK,OAAO,YAAY,SAAS,EAAE,CAAC,MAAM;CACzE,MAAM,YAAoB;mBACT,UAAU,KAAK,iBAAiB,WAAW;CAC5D,MAAM,aAAa,GAAG,QAAQ,UAAU;CACxC,MAAM,kBAAkB,GAAG,QACzB,YAAY,IAAI,YAAY,OAAA,GAA6B,CAC1D;CAED,MAAM,EAAC,WAAU,uBACf,OACA,aACA,YACA,gBACD;CACD,MAAM,eAAe,YAAY;CACjC,MAAM,iBAAiB,eAAA;CAEvB,MAAM,gBAAiC,MAAM,KAAK,EAChD,QAAQ,oBAAoB,cAC7B,CAAC;CACF,IAAI,cAAc;CAClB,IAAI,cAAc;CAElB,SAAS,QAAQ;EACf,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,cAAc;EACpB,MAAM,cAAc;EAEpB,IAAI,IAAI;AACR,SAAO,cAAA,IAAiC,eAAA,GACtC,iBAAgB,IAAI,cAAc,MAAM,GAAI,KAAK,eAAgB,CAAC;AAEpE,SAAO,cAAc,GAAG,cACtB,YAAW,IAAI,cAAc,MAAM,GAAI,KAAK,aAAc,CAAC;EAE7D,MAAM,gBAAgB,cAAc;AACpC,OAAK,IAAI,IAAI,GAAG,IAAI,eAAe,IACjC,eAAc,KAAK,KAAA;AAErB,gBAAc;AACd,SAAO,QAAQ;EAEf,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,eAAa;AACb,KAAG,QACD,WAAW,YAAY,GAAG,UAAU,SAAS,YAAY,aAAa,QAAQ,QAAQ,EAAE,CAAC,KAC1F;;AAGH,IAAG,OAAO,gCAAgC,UAAU,IAAI,OAAO;CAC/D,MAAM,YAAY,MAAM,eAAe,UAAU,EAAC,oBAAoB,MAAK,CAAC;CAC5E,MAAM,UAAU,YAAY,KAAI,MAAK;EACnC,MAAM,UAAU,UAAU,cAAc,EAAE,QAAQ;AAClD,UAAQ,QACN,UACE,QAAQ,IAAI,EACZ,EAAE,UAAA,IAEH;GACH;CAEF,MAAM,YAAY,IAAI,WAAW;CACjC,IAAI,MAAM;AAEV,OAAM,WACJ,MAAM,KAAK,OAAO,SAAS,OAAO,aAAa,CAAC,UAAU,EAC1D,IAAI,SAAS;EACX,eAAe;EAEf,MACE,OACA,WACA,UACA;AACA,OAAI;AACF,SAAK,MAAM,QAAQ,UAAU,MAAM,MAAM,EAAE;AACzC,oBAAe,SAAS,OAAO,IAAI,KAAK;AACxC,mBAAc,cAAc,eAAe,OACzC,SAAS,OAAO,OAAO,QAAQ,KAAK,KAAK;AAE3C,SAAI,EAAE,QAAQ,QAAQ,QAAQ;AAC5B,YAAM;AACN,UACE,EAAE,eAAe,oBAAoB,gBACrC,eAAe,wBAEf,QAAO;;;AAIb,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAI3D,QAAQ,aAAsC;AAC5C,OAAI;AACF,WAAO;AACP,cAAU;YACH,GAAG;AACV,aAAS,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,EAAE,CAAC,CAAC;;;EAG5D,CAAC,CACH;CAED,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,IAAG,OACD,oBAAoB,OAAO,KAAK,aAAa,UAAU,WAC1C,UAAU,QAAQ,EAAE,CAAC,eAAe,QAAQ,QAAQ,EAAE,CAAC,OACrE;AACD,QAAO;EAAC,MAAM,OAAO;EAAM;EAAU"}

package/out/zero-cache/src/services/replicator/write-worker-client.d.ts CHANGED Viewed

@@ -19,6 +19,15 @@ export interface WriteWorkerClient {
     stop(): Promise<void>;
     onError(handler: ErrorHandler): void;
 }
+export type SerializedError = {
+    name: string;
+    message: string;
+    stack?: string | undefined;
+    cause?: SerializedError | string | undefined;
+    details?: Record<string, unknown> | undefined;
+};
+export declare function serializeError(err: unknown): SerializedError;
+export declare function deserializeError(serialized: SerializedError): Error;
 export type ArgsMap = {
     init: [string, ChangeProcessorMode, PragmaConfig, LogConfig];
     getSubscriptionState: [];
@@ -44,11 +53,11 @@ export type Response<M extends Method = Method> = {
     error?: undefined;
 } | {
     method: M;
-    error: unknown;
+    error: SerializedError;
     result?: undefined;
 };
 export type WriteError = {
-    writeError: Error;
+    writeError: SerializedError;
 };
 export declare function applyPragmas(db: Database, pragmas: PragmaConfig): void;
 /**

package/out/zero-cache/src/services/replicator/write-worker-client.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"write-worker-client.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAE3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,mBAAmB,EAAE,YAAY,EAAC,MAAM,uBAAuB,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAErE,MAAM,MAAM,YAAY,GAAG;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC,CAAC;AAEF,KAAK,YAAY,GAAG,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,oBAAoB,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACnD,cAAc,CAAC,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAC3E,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC;CACtC;AAGD,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,CAAC,MAAM,EAAE,mBAAmB,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;IAC7D,oBAAoB,EAAE,EAAE,CAAC;IACzB,cAAc,EAAE,CAAC,gBAAgB,CAAC,CAAC;IACnC,KAAK,EAAE,EAAE,CAAC;IACV,IAAI,EAAE,EAAE,CAAC;CACV,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;AAEnC,MAAM,MAAM,OAAO,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;CAAC,CAAC;AAE/E,MAAM,MAAM,SAAS,GAAG;IACtB,IAAI,EAAE,IAAI,CAAC;IACX,oBAAoB,EAAE,iBAAiB,CAAC;IACxC,cAAc,EAAE,YAAY,GAAG,IAAI,CAAC;IACpC,KAAK,EAAE,IAAI,CAAC;IACZ,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,QAAQ,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAC1C;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,SAAS,CAAA;CAAC,GACpD;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,KAAK,EAAE,~~OAAO~~,CAAC;IAAC,MAAM,CAAC,EAAE,SAAS,CAAA;CAAC,CAAC;~~AAEpD~~,MAAM,MAAM,UAAU,GAAG;IAAC,UAAU,EAAE,~~KAAK~~,CAAA;CAAC,CAAC;~~AAE7C~~,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,QAM/D;AAED;;;GAGG;AACH,qBAAa,uBAAwB,YAAW,iBAAiB;;;~~IA8D~~/D,IAAI,CACF,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,mBAAmB,EACzB,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,SAAS,GACnB,OAAO,CAAC,IAAI,CAAC;IAIhB,oBAAoB,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAIlD,cAAc,CAAC,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAI1E,KAAK,IAAI,IAAI;IAMP,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;CAGrC"}
1	+ {"version":3,"file":"write-worker-client.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAE3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,mBAAmB,EAAE,YAAY,EAAC,MAAM,uBAAuB,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAErE,MAAM,MAAM,YAAY,GAAG;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC,CAAC;AAEF,KAAK,YAAY,GAAG,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,oBAAoB,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACnD,cAAc,CAAC,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAC3E,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,KAAK,CAAC,EAAE,eAAe,GAAG,MAAM,GAAG,SAAS,CAAC;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CAC/C,CAAC;AAEF,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,eAAe,CA8B5D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,eAAe,GAAG,KAAK,CAgBnE;AAGD,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,CAAC,MAAM,EAAE,mBAAmB,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;IAC7D,oBAAoB,EAAE,EAAE,CAAC;IACzB,cAAc,EAAE,CAAC,gBAAgB,CAAC,CAAC;IACnC,KAAK,EAAE,EAAE,CAAC;IACV,IAAI,EAAE,EAAE,CAAC;CACV,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;AAEnC,MAAM,MAAM,OAAO,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;CAAC,CAAC;AAE/E,MAAM,MAAM,SAAS,GAAG;IACtB,IAAI,EAAE,IAAI,CAAC;IACX,oBAAoB,EAAE,iBAAiB,CAAC;IACxC,cAAc,EAAE,YAAY,GAAG,IAAI,CAAC;IACpC,KAAK,EAAE,IAAI,CAAC;IACZ,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,QAAQ,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAC1C;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,SAAS,CAAA;CAAC,GACpD;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,KAAK,EAAE,eAAe,CAAC;IAAC,MAAM,CAAC,EAAE,SAAS,CAAA;CAAC,CAAC;AAE5D,MAAM,MAAM,UAAU,GAAG;IAAC,UAAU,EAAE,eAAe,CAAA;CAAC,CAAC;AAEvD,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,QAM/D;AAED;;;GAGG;AACH,qBAAa,uBAAwB,YAAW,iBAAiB;;;IAyD/D,IAAI,CACF,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,mBAAmB,EACzB,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,SAAS,GACnB,OAAO,CAAC,IAAI,CAAC;IAIhB,oBAAoB,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAIlD,cAAc,CAAC,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAI1E,KAAK,IAAI,IAAI;IAMP,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;CAGrC"}

package/out/zero-cache/src/services/replicator/write-worker-client.js CHANGED Viewed

@@ -3,6 +3,35 @@ import { WRITE_WORKER_URL } from "../../server/worker-urls.js";
 import { resolver } from "@rocicorp/resolver";
 import { Worker } from "node:worker_threads";
 //#region ../zero-cache/src/services/replicator/write-worker-client.ts
+function serializeError(err) {
+	if (!(err instanceof Error)) return {
+		name: "Error",
+		message: String(err),
+		details: err && typeof err === "object" ? { ...err } : void 0
+	};
+	const details = Object.fromEntries(Object.getOwnPropertyNames(err).filter((key) => ![
+		"name",
+		"message",
+		"stack",
+		"cause"
+	].includes(key)).map((key) => [key, err[key]]));
+	const cause = err.cause instanceof Error ? serializeError(err.cause) : err.cause === void 0 ? void 0 : String(err.cause);
+	return {
+		name: err.name,
+		message: err.message,
+		stack: err.stack,
+		cause,
+		details: Object.keys(details).length ? details : void 0
+	};
+}
+function deserializeError(serialized) {
+	const err = new Error(serialized.message);
+	err.name = serialized.name;
+	if (serialized.stack !== void 0) err.stack = serialized.stack;
+	if (serialized.cause !== void 0) err.cause = typeof serialized.cause === "string" ? serialized.cause : deserializeError(serialized.cause);
+	if (serialized.details) Object.assign(err, serialized.details);
+	return err;
+}
 function applyPragmas(db, pragmas) {
 	db.pragma(`busy_timeout = ${pragmas.busyTimeout}`);
 	db.pragma(`analysis_limit = ${pragmas.analysisLimit}`);
@@ -21,7 +50,7 @@ var ThreadWriteWorkerClient = class {
 		this.#worker = new Worker(WRITE_WORKER_URL);
 		this.#worker.on("message", (msg) => {
 			if ("writeError" in msg) {
-				const error = msg.writeError instanceof Error ? msg.writeError : new Error(String(msg.writeError));
+				const error = deserializeError(msg.writeError);
 				this.#rejectAll(error);
 				this.#errorHandler(error);
 				return;
@@ -29,7 +58,7 @@ var ThreadWriteWorkerClient = class {
 			const r = this.#pending;
 			if (!r) return;
 			this.#pending = null;
-			if (msg.error !== void 0) r.reject(msg.error instanceof Error ? msg.error : new Error(String(msg.error)));
+			if (msg.error !== void 0) r.reject(deserializeError(msg.error));
 			else r.resolve(msg.result);
 		});
 		this.#worker.on("error", (err) => {
@@ -91,6 +120,6 @@ var ThreadWriteWorkerClient = class {
 	}
 };
 //#endregion
-export { ThreadWriteWorkerClient, applyPragmas };
+export { ThreadWriteWorkerClient, applyPragmas, serializeError };
 //# sourceMappingURL=write-worker-client.js.map

package/out/zero-cache/src/services/replicator/write-worker-client.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"write-worker-client.js","names":["#worker","#rejectAll","#errorHandler","#pending","#terminated","#call"],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"sourcesContent":["import {Worker} from 'node:worker_threads';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {WRITE_WORKER_URL} from '../../server/worker-urls.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeProcessorMode, CommitResult} from './change-processor.ts';\nimport type {SubscriptionState} from './schema/replication-state.ts';\n\nexport type PragmaConfig = {\n busyTimeout: number;\n analysisLimit: number;\n walAutocheckpoint?: number \| undefined;\n};\n\ntype ErrorHandler = (err: Error) => void;\n\n/*\n Interface for a write worker that processes replication messages.\n /\nexport interface WriteWorkerClient {\n getSubscriptionState(): Promise<SubscriptionState>;\n processMessage(downstream: ChangeStreamData): Promise<CommitResult \| null>;\n abort(): void;\n stop(): Promise<void>;\n onError(handler: ErrorHandler): void;\n}\n\n// ~~Wire~~ ~~protocol~~ ~~types~~ — ~~errors~~ are ~~passed~~ ~~directly~~ ~~via~~ ~~structured~~ ~~clone~~\nexport type ArgsMap = {\n init: [string, ChangeProcessorMode, PragmaConfig, LogConfig];\n getSubscriptionState: [];\n processMessage: [ChangeStreamData];\n abort: [];\n stop: [];\n};\n\nexport type Method = keyof ArgsMap;\n\nexport type Request<M extends Method = Method> = {method: M; args: ArgsMap[M]};\n\nexport type ResultMap = {\n init: void;\n getSubscriptionState: SubscriptionState;\n processMessage: CommitResult \| null;\n abort: void;\n stop: void;\n};\n\nexport type Response<M extends Method = Method> =\n \| {method: M; result: ResultMap[M]; error?: undefined}\n \| {method: M; error: ~~unknown~~; result?: undefined};\n\nexport type WriteError = {writeError: ~~Error~~};\n\nexport function applyPragmas(db: Database, pragmas: PragmaConfig) {\n db.pragma(`busy_timeout = ${pragmas.busyTimeout}`);\n db.pragma(`analysis_limit = ${pragmas.analysisLimit}`);\n if (pragmas.walAutocheckpoint !== undefined) {\n db.pragma(`wal_autocheckpoint = ${pragmas.walAutocheckpoint}`);\n }\n}\n\n/\n Delegates SQLite writes to a worker_thread,\n * keeping the main event loop free for WebSocket heartbeats and IPC.\n */\nexport class ThreadWriteWorkerClient implements WriteWorkerClient {\n readonly #worker: Worker;\n #pending: Resolver<unknown, Error> \| null = null;\n #errorHandler: ErrorHandler = () => {};\n #terminated = false;\n\n constructor() {\n this.#worker = new Worker(WRITE_WORKER_URL);\n\n this.#worker.on('message', (msg: Response \| WriteError) => {\n if ('writeError' in msg) {\n const error ~~=\n msg.writeError~~ ~~instanceof Error\n ? msg.writeError\n : new Error~~(~~String(~~msg.writeError));\n this.#rejectAll(error);\n this.#errorHandler(error);\n return;\n }\n const r = this.#pending;\n if (!r) return; // stale abort response\n this.#pending = null;\n if (msg.error !== undefined) {\n r.reject(~~\n msg.error instanceof Error ? msg.error : new Error~~(~~String(~~msg.error))~~,\n )~~;\n } else {\n r.resolve(msg.result);\n }\n });\n\n this.#worker.on('error', (err: Error) => {\n this.#rejectAll(err);\n this.#errorHandler(err);\n });\n\n this.#worker.on('exit', (code: number) => {\n this.#terminated = true;\n if (code !== 0) {\n const err = new Error(`Worker exited with code ${code}`);\n this.#rejectAll(err);\n this.#errorHandler(err);\n }\n });\n }\n\n #rejectAll(err: Error) {\n const r = this.#pending;\n if (r) {\n this.#pending = null;\n r.reject(err);\n }\n }\n\n #call<M extends Method>(method: M, args: ArgsMap[M]): Promise<ResultMap[M]> {\n assert(this.#pending === null, `concurrent call: ${method}`);\n const r = resolver<ResultMap[M]>();\n this.#pending = r as Resolver<unknown, Error>;\n this.#worker.postMessage({method, args} satisfies Request);\n return r.promise;\n }\n\n init(\n dbPath: string,\n mode: ChangeProcessorMode,\n pragmas: PragmaConfig,\n logConfig: LogConfig,\n ): Promise<void> {\n return this.#call('init', [dbPath, mode, pragmas, logConfig]);\n }\n\n getSubscriptionState(): Promise<SubscriptionState> {\n return this.#call('getSubscriptionState', []);\n }\n\n processMessage(downstream: ChangeStreamData): Promise<CommitResult \| null> {\n return this.#call('processMessage', [downstream]);\n }\n\n abort(): void {\n if (!this.#terminated) {\n this.#worker.postMessage({method: 'abort', args: []} satisfies Request);\n }\n }\n\n async stop(): Promise<void> {\n await this.#call('stop', []);\n if (!this.#terminated) {\n await this.#worker.terminate();\n }\n }\n\n onError(handler: ErrorHandler): void {\n this.#errorHandler = handler;\n }\n}\n"],"mappings":";;;;;~~AAwDA~~,SAAgB,aAAa,IAAc,SAAuB;AAChE,IAAG,OAAO,kBAAkB,QAAQ,cAAc;AAClD,IAAG,OAAO,oBAAoB,QAAQ,gBAAgB;AACtD,KAAI,QAAQ,sBAAsB,KAAA,EAChC,IAAG,OAAO,wBAAwB,QAAQ,oBAAoB;;;;;;AAQlE,IAAa,0BAAb,MAAkE;CAChE;CACA,WAA4C;CAC5C,sBAAoC;CACpC,cAAc;CAEd,cAAc;AACZ,QAAA,SAAe,IAAI,OAAO,iBAAiB;AAE3C,QAAA,OAAa,GAAG,YAAY,QAA+B;AACzD,OAAI,gBAAgB,KAAK;IACvB,MAAM,~~QACJ~~,~~IAAI~~,~~sBAAsB,QACtB,~~IAAI,~~aACJ,IAAI,MAAM,OAAO,IAAI,~~WAAW~~,CAAC~~;~~AACvC~~,UAAA,UAAgB,MAAM;AACtB,UAAA,aAAmB,MAAM;AACzB;;GAEF,MAAM,IAAI,MAAA;AACV,OAAI,CAAC,EAAG;AACR,SAAA,UAAgB;AAChB,OAAI,IAAI,UAAU,KAAA,EAChB,GAAE,~~OACA~~,~~IAAI,~~iBAAiB,~~QAAQ,~~IAAI,~~QAAQ,IAAI,~~MAAM,~~OAAO,IAAI,MAAM,~~CAAC~~,CACtE~~;~~OAED~~,GAAE,QAAQ,IAAI,OAAO;IAEvB;AAEF,QAAA,OAAa,GAAG,UAAU,QAAe;AACvC,SAAA,UAAgB,IAAI;AACpB,SAAA,aAAmB,IAAI;IACvB;AAEF,QAAA,OAAa,GAAG,SAAS,SAAiB;AACxC,SAAA,aAAmB;AACnB,OAAI,SAAS,GAAG;IACd,MAAM,sBAAM,IAAI,MAAM,2BAA2B,OAAO;AACxD,UAAA,UAAgB,IAAI;AACpB,UAAA,aAAmB,IAAI;;IAEzB;;CAGJ,WAAW,KAAY;EACrB,MAAM,IAAI,MAAA;AACV,MAAI,GAAG;AACL,SAAA,UAAgB;AAChB,KAAE,OAAO,IAAI;;;CAIjB,MAAwB,QAAW,MAAyC;AAC1E,SAAO,MAAA,YAAkB,MAAM,oBAAoB,SAAS;EAC5D,MAAM,IAAI,UAAwB;AAClC,QAAA,UAAgB;AAChB,QAAA,OAAa,YAAY;GAAC;GAAQ;GAAK,CAAmB;AAC1D,SAAO,EAAE;;CAGX,KACE,QACA,MACA,SACA,WACe;AACf,SAAO,MAAA,KAAW,QAAQ;GAAC;GAAQ;GAAM;GAAS;GAAU,CAAC;;CAG/D,uBAAmD;AACjD,SAAO,MAAA,KAAW,wBAAwB,EAAE,CAAC;;CAG/C,eAAe,YAA4D;AACzE,SAAO,MAAA,KAAW,kBAAkB,CAAC,WAAW,CAAC;;CAGnD,QAAc;AACZ,MAAI,CAAC,MAAA,WACH,OAAA,OAAa,YAAY;GAAC,QAAQ;GAAS,MAAM,EAAE;GAAC,CAAmB;;CAI3E,MAAM,OAAsB;AAC1B,QAAM,MAAA,KAAW,QAAQ,EAAE,CAAC;AAC5B,MAAI,CAAC,MAAA,WACH,OAAM,MAAA,OAAa,WAAW;;CAIlC,QAAQ,SAA6B;AACnC,QAAA,eAAqB"}
1	+ {"version":3,"file":"write-worker-client.js","names":["#worker","#rejectAll","#errorHandler","#pending","#terminated","#call"],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"sourcesContent":["import {Worker} from 'node:worker_threads';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {WRITE_WORKER_URL} from '../../server/worker-urls.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeProcessorMode, CommitResult} from './change-processor.ts';\nimport type {SubscriptionState} from './schema/replication-state.ts';\n\nexport type PragmaConfig = {\n busyTimeout: number;\n analysisLimit: number;\n walAutocheckpoint?: number \| undefined;\n};\n\ntype ErrorHandler = (err: Error) => void;\n\n/*\n Interface for a write worker that processes replication messages.\n /\nexport interface WriteWorkerClient {\n getSubscriptionState(): Promise<SubscriptionState>;\n processMessage(downstream: ChangeStreamData): Promise<CommitResult \| null>;\n abort(): void;\n stop(): Promise<void>;\n onError(handler: ErrorHandler): void;\n}\n\nexport type SerializedError = {\n name: string;\n message: string;\n stack?: string \| undefined;\n cause?: SerializedError \| string \| undefined;\n details?: Record<string, unknown> \| undefined;\n};\n\nexport function serializeError(err: unknown): SerializedError {\n if (!(err instanceof Error)) {\n return {\n name: 'Error',\n message: String(err),\n details: err && typeof err === 'object' ? {...err} : undefined,\n };\n }\n\n // Error fields such as message, stack, and some native error details are\n // non-enumerable, so JSON.stringify(err) would usually return \"{}\".\n const details = Object.fromEntries(\n Object.getOwnPropertyNames(err)\n .filter(key => !['name', 'message', 'stack', 'cause'].includes(key))\n .map(key => [key, (err as unknown as Record<string, unknown>)[key]]),\n );\n const cause =\n err.cause instanceof Error\n ? serializeError(err.cause)\n : err.cause === undefined\n ? undefined\n : String(err.cause);\n\n return {\n name: err.name,\n message: err.message,\n stack: err.stack,\n cause,\n details: Object.keys(details).length ? details : undefined,\n };\n}\n\nexport function deserializeError(serialized: SerializedError): Error {\n const err = new Error(serialized.message);\n err.name = serialized.name;\n if (serialized.stack !== undefined) {\n err.stack = serialized.stack;\n }\n if (serialized.cause !== undefined) {\n err.cause =\n typeof serialized.cause === 'string'\n ? serialized.cause\n : deserializeError(serialized.cause);\n }\n if (serialized.details) {\n Object.assign(err, serialized.details);\n }\n return err;\n}\n\n// Wire protocol types.\nexport type ArgsMap = {\n init: [string, ChangeProcessorMode, PragmaConfig, LogConfig];\n getSubscriptionState: [];\n processMessage: [ChangeStreamData];\n abort: [];\n stop: [];\n};\n\nexport type Method = keyof ArgsMap;\n\nexport type Request<M extends Method = Method> = {method: M; args: ArgsMap[M]};\n\nexport type ResultMap = {\n init: void;\n getSubscriptionState: SubscriptionState;\n processMessage: CommitResult \| null;\n abort: void;\n stop: void;\n};\n\nexport type Response<M extends Method = Method> =\n \| {method: M; result: ResultMap[M]; error?: undefined}\n \| {method: M; error: SerializedError; result?: undefined};\n\nexport type WriteError = {writeError: SerializedError};\n\nexport function applyPragmas(db: Database, pragmas: PragmaConfig) {\n db.pragma(`busy_timeout = ${pragmas.busyTimeout}`);\n db.pragma(`analysis_limit = ${pragmas.analysisLimit}`);\n if (pragmas.walAutocheckpoint !== undefined) {\n db.pragma(`wal_autocheckpoint = ${pragmas.walAutocheckpoint}`);\n }\n}\n\n/\n Delegates SQLite writes to a worker_thread,\n * keeping the main event loop free for WebSocket heartbeats and IPC.\n */\nexport class ThreadWriteWorkerClient implements WriteWorkerClient {\n readonly #worker: Worker;\n #pending: Resolver<unknown, Error> \| null = null;\n #errorHandler: ErrorHandler = () => {};\n #terminated = false;\n\n constructor() {\n this.#worker = new Worker(WRITE_WORKER_URL);\n\n this.#worker.on('message', (msg: Response \| WriteError) => {\n if ('writeError' in msg) {\n const error = deserializeError(msg.writeError);\n this.#rejectAll(error);\n this.#errorHandler(error);\n return;\n }\n const r = this.#pending;\n if (!r) return; // stale abort response\n this.#pending = null;\n if (msg.error !== undefined) {\n r.reject(deserializeError(msg.error));\n } else {\n r.resolve(msg.result);\n }\n });\n\n this.#worker.on('error', (err: Error) => {\n this.#rejectAll(err);\n this.#errorHandler(err);\n });\n\n this.#worker.on('exit', (code: number) => {\n this.#terminated = true;\n if (code !== 0) {\n const err = new Error(`Worker exited with code ${code}`);\n this.#rejectAll(err);\n this.#errorHandler(err);\n }\n });\n }\n\n #rejectAll(err: Error) {\n const r = this.#pending;\n if (r) {\n this.#pending = null;\n r.reject(err);\n }\n }\n\n #call<M extends Method>(method: M, args: ArgsMap[M]): Promise<ResultMap[M]> {\n assert(this.#pending === null, `concurrent call: ${method}`);\n const r = resolver<ResultMap[M]>();\n this.#pending = r as Resolver<unknown, Error>;\n this.#worker.postMessage({method, args} satisfies Request);\n return r.promise;\n }\n\n init(\n dbPath: string,\n mode: ChangeProcessorMode,\n pragmas: PragmaConfig,\n logConfig: LogConfig,\n ): Promise<void> {\n return this.#call('init', [dbPath, mode, pragmas, logConfig]);\n }\n\n getSubscriptionState(): Promise<SubscriptionState> {\n return this.#call('getSubscriptionState', []);\n }\n\n processMessage(downstream: ChangeStreamData): Promise<CommitResult \| null> {\n return this.#call('processMessage', [downstream]);\n }\n\n abort(): void {\n if (!this.#terminated) {\n this.#worker.postMessage({method: 'abort', args: []} satisfies Request);\n }\n }\n\n async stop(): Promise<void> {\n await this.#call('stop', []);\n if (!this.#terminated) {\n await this.#worker.terminate();\n }\n }\n\n onError(handler: ErrorHandler): void {\n this.#errorHandler = handler;\n }\n}\n"],"mappings":";;;;;AAqCA,SAAgB,eAAe,KAA+B;AAC5D,KAAI,EAAE,eAAe,OACnB,QAAO;EACL,MAAM;EACN,SAAS,OAAO,IAAI;EACpB,SAAS,OAAO,OAAO,QAAQ,WAAW,EAAC,GAAG,KAAI,GAAG,KAAA;EACtD;CAKH,MAAM,UAAU,OAAO,YACrB,OAAO,oBAAoB,IAAI,CAC5B,QAAO,QAAO,CAAC;EAAC;EAAQ;EAAW;EAAS;EAAQ,CAAC,SAAS,IAAI,CAAC,CACnE,KAAI,QAAO,CAAC,KAAM,IAA2C,KAAK,CAAC,CACvE;CACD,MAAM,QACJ,IAAI,iBAAiB,QACjB,eAAe,IAAI,MAAM,GACzB,IAAI,UAAU,KAAA,IACZ,KAAA,IACA,OAAO,IAAI,MAAM;AAEzB,QAAO;EACL,MAAM,IAAI;EACV,SAAS,IAAI;EACb,OAAO,IAAI;EACX;EACA,SAAS,OAAO,KAAK,QAAQ,CAAC,SAAS,UAAU,KAAA;EAClD;;AAGH,SAAgB,iBAAiB,YAAoC;CACnE,MAAM,MAAM,IAAI,MAAM,WAAW,QAAQ;AACzC,KAAI,OAAO,WAAW;AACtB,KAAI,WAAW,UAAU,KAAA,EACvB,KAAI,QAAQ,WAAW;AAEzB,KAAI,WAAW,UAAU,KAAA,EACvB,KAAI,QACF,OAAO,WAAW,UAAU,WACxB,WAAW,QACX,iBAAiB,WAAW,MAAM;AAE1C,KAAI,WAAW,QACb,QAAO,OAAO,KAAK,WAAW,QAAQ;AAExC,QAAO;;AA8BT,SAAgB,aAAa,IAAc,SAAuB;AAChE,IAAG,OAAO,kBAAkB,QAAQ,cAAc;AAClD,IAAG,OAAO,oBAAoB,QAAQ,gBAAgB;AACtD,KAAI,QAAQ,sBAAsB,KAAA,EAChC,IAAG,OAAO,wBAAwB,QAAQ,oBAAoB;;;;;;AAQlE,IAAa,0BAAb,MAAkE;CAChE;CACA,WAA4C;CAC5C,sBAAoC;CACpC,cAAc;CAEd,cAAc;AACZ,QAAA,SAAe,IAAI,OAAO,iBAAiB;AAE3C,QAAA,OAAa,GAAG,YAAY,QAA+B;AACzD,OAAI,gBAAgB,KAAK;IACvB,MAAM,QAAQ,iBAAiB,IAAI,WAAW;AAC9C,UAAA,UAAgB,MAAM;AACtB,UAAA,aAAmB,MAAM;AACzB;;GAEF,MAAM,IAAI,MAAA;AACV,OAAI,CAAC,EAAG;AACR,SAAA,UAAgB;AAChB,OAAI,IAAI,UAAU,KAAA,EAChB,GAAE,OAAO,iBAAiB,IAAI,MAAM,CAAC;OAErC,GAAE,QAAQ,IAAI,OAAO;IAEvB;AAEF,QAAA,OAAa,GAAG,UAAU,QAAe;AACvC,SAAA,UAAgB,IAAI;AACpB,SAAA,aAAmB,IAAI;IACvB;AAEF,QAAA,OAAa,GAAG,SAAS,SAAiB;AACxC,SAAA,aAAmB;AACnB,OAAI,SAAS,GAAG;IACd,MAAM,sBAAM,IAAI,MAAM,2BAA2B,OAAO;AACxD,UAAA,UAAgB,IAAI;AACpB,UAAA,aAAmB,IAAI;;IAEzB;;CAGJ,WAAW,KAAY;EACrB,MAAM,IAAI,MAAA;AACV,MAAI,GAAG;AACL,SAAA,UAAgB;AAChB,KAAE,OAAO,IAAI;;;CAIjB,MAAwB,QAAW,MAAyC;AAC1E,SAAO,MAAA,YAAkB,MAAM,oBAAoB,SAAS;EAC5D,MAAM,IAAI,UAAwB;AAClC,QAAA,UAAgB;AAChB,QAAA,OAAa,YAAY;GAAC;GAAQ;GAAK,CAAmB;AAC1D,SAAO,EAAE;;CAGX,KACE,QACA,MACA,SACA,WACe;AACf,SAAO,MAAA,KAAW,QAAQ;GAAC;GAAQ;GAAM;GAAS;GAAU,CAAC;;CAG/D,uBAAmD;AACjD,SAAO,MAAA,KAAW,wBAAwB,EAAE,CAAC;;CAG/C,eAAe,YAA4D;AACzE,SAAO,MAAA,KAAW,kBAAkB,CAAC,WAAW,CAAC;;CAGnD,QAAc;AACZ,MAAI,CAAC,MAAA,WACH,OAAA,OAAa,YAAY;GAAC,QAAQ;GAAS,MAAM,EAAE;GAAC,CAAmB;;CAI3E,MAAM,OAAsB;AAC1B,QAAM,MAAA,KAAW,QAAQ,EAAE,CAAC;AAC5B,MAAI,CAAC,MAAA,WACH,OAAM,MAAA,OAAa,WAAW;;CAIlC,QAAQ,SAA6B;AACnC,QAAA,eAAqB"}

package/out/zero-cache/src/services/replicator/write-worker.js CHANGED Viewed

@@ -4,7 +4,7 @@ import { getSubscriptionState } from "./schema/replication-state.js";
 import { StatementRunner } from "../../db/statements.js";
 import { createLogContext } from "../../server/logging.js";
 import { ChangeProcessor } from "./change-processor.js";
-import { applyPragmas } from "./write-worker-client.js";
+import { applyPragmas, serializeError } from "./write-worker-client.js";
 import { parentPort } from "node:worker_threads";
 //#region ../zero-cache/src/services/replicator/write-worker.ts
 if (!parentPort) throw new Error("write-worker must be run as a worker thread");
@@ -17,7 +17,7 @@ function createAPI() {
 	let lc;
 	function createProcessor() {
 		processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {
-			port.postMessage({ writeError: err instanceof Error ? err : new Error(String(err)) });
+			port.postMessage({ writeError: serializeError(err) });
 		});
 	}
 	return {
@@ -58,7 +58,7 @@ port.on("message", (msg) => {
 	} catch (e) {
 		if (msg.method !== "abort") port.postMessage({
 			method: msg.method,
-			error: e
+			error: serializeError(e)
 		});
 	}
 });

package/out/zero-cache/src/services/replicator/write-worker.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"write-worker.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker.ts"],"sourcesContent":["import {parentPort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {createLogContext} from '../../server/logging.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {ChangeProcessor, type ChangeProcessorMode} from './change-processor.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport {\n applyPragmas,\n type ArgsMap,\n type Method,\n type PragmaConfig,\n type Request,\n type Response,\n type ResultMap,\n type WriteError,\n} from './write-worker-client.ts';\n\nif (!parentPort) {\n throw new Error('write-worker must be run as a worker thread');\n}\n\nconst port = parentPort;\n\ntype API = {[M in Method]: (...args: ArgsMap[M]) => ResultMap[M]};\n\nfunction createAPI(): API {\n let db: Database \| undefined;\n let runner: StatementRunner \| undefined;\n let processor: ChangeProcessor \| undefined;\n let mode: ChangeProcessorMode \| undefined;\n let lc: LogContext \| undefined;\n\n function createProcessor() {\n processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {\n port.postMessage({\n writeError: ~~err instanceof Error ? err : new Error~~(~~String(~~err)),\n } satisfies WriteError);\n });\n }\n\n return {\n init(\n dbPath: string,\n cpMode: ChangeProcessorMode,\n pragmas: PragmaConfig,\n logConfig: LogConfig,\n ) {\n lc = createLogContext({log: logConfig}, 'write-worker');\n db = new Database(lc, dbPath);\n applyPragmas(db, pragmas);\n runner = new StatementRunner(db);\n mode = cpMode;\n createProcessor();\n },\n\n getSubscriptionState() {\n return getSubscriptionState(must(runner));\n },\n\n processMessage(downstream: ChangeStreamData) {\n return must(processor).processMessage(must(lc), downstream);\n },\n\n abort() {\n must(processor).abort(must(lc));\n createProcessor();\n },\n\n stop() {\n db?.close();\n db = undefined;\n runner = undefined;\n processor = undefined;\n },\n };\n}\n\nconst api = createAPI();\n\nport.on('message', (msg: Request) => {\n try {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TS can't narrow msg.method + msg.args together\n const result = (api[msg.method] as (...args: any[]) => unknown)(\n ...msg.args,\n );\n // abort is fire-and-forget — no pending slot on the client side.\n if (msg.method !== 'abort') {\n port.postMessage({method: msg.method, result} as Response);\n }\n } catch (e) {\n if (msg.method !== 'abort') {\n port.postMessage({method: msg.method, error: e} as Response);\n }\n }\n});\n"],"mappings":";;;;;;;;;~~AAqBA~~,IAAI,CAAC,WACH,OAAM,IAAI,MAAM,8CAA8C;AAGhE,IAAM,OAAO;AAIb,SAAS,YAAiB;CACxB,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CAEJ,SAAS,kBAAkB;AACzB,cAAY,IAAI,gBAAgB,KAAK,OAAO,EAAE,KAAK,KAAK,GAAG,KAAK,QAAQ;AACtE,QAAK,YAAY,EACf,YAAY,eAAe,~~QAAQ,MAAM,~~IAAI,~~MAAM~~,~~OAAO,IAAI,CAAC,EAChE,~~CAAsB;IACvB;;AAGJ,QAAO;EACL,KACE,QACA,QACA,SACA,WACA;AACA,QAAK,iBAAiB,EAAC,KAAK,WAAU,EAAE,eAAe;AACvD,QAAK,IAAI,SAAS,IAAI,OAAO;AAC7B,gBAAa,IAAI,QAAQ;AACzB,YAAS,IAAI,gBAAgB,GAAG;AAChC,UAAO;AACP,oBAAiB;;EAGnB,uBAAuB;AACrB,UAAO,qBAAqB,KAAK,OAAO,CAAC;;EAG3C,eAAe,YAA8B;AAC3C,UAAO,KAAK,UAAU,CAAC,eAAe,KAAK,GAAG,EAAE,WAAW;;EAG7D,QAAQ;AACN,QAAK,UAAU,CAAC,MAAM,KAAK,GAAG,CAAC;AAC/B,oBAAiB;;EAGnB,OAAO;AACL,OAAI,OAAO;AACX,QAAK,KAAA;AACL,YAAS,KAAA;AACT,eAAY,KAAA;;EAEf;;AAGH,IAAM,MAAM,WAAW;AAEvB,KAAK,GAAG,YAAY,QAAiB;AACnC,KAAI;EAEF,MAAM,SAAU,IAAI,IAAI,QACtB,GAAG,IAAI,KACR;AAED,MAAI,IAAI,WAAW,QACjB,MAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ;GAAO,CAAa;UAErD,GAAG;AACV,MAAI,IAAI,WAAW,QACjB,MAAK,YAAY;~~GAAC~~,QAAQ,IAAI;~~GAAQ~~,OAAO;~~GAAE~~,CAAa;;~~EAGhE~~"}
1	+ {"version":3,"file":"write-worker.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker.ts"],"sourcesContent":["import {parentPort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {createLogContext} from '../../server/logging.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {ChangeProcessor, type ChangeProcessorMode} from './change-processor.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport {\n applyPragmas,\n serializeError,\n type ArgsMap,\n type Method,\n type PragmaConfig,\n type Request,\n type Response,\n type ResultMap,\n type WriteError,\n} from './write-worker-client.ts';\n\nif (!parentPort) {\n throw new Error('write-worker must be run as a worker thread');\n}\n\nconst port = parentPort;\n\ntype API = {[M in Method]: (...args: ArgsMap[M]) => ResultMap[M]};\n\nfunction createAPI(): API {\n let db: Database \| undefined;\n let runner: StatementRunner \| undefined;\n let processor: ChangeProcessor \| undefined;\n let mode: ChangeProcessorMode \| undefined;\n let lc: LogContext \| undefined;\n\n function createProcessor() {\n processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {\n port.postMessage({\n writeError: serializeError(err),\n } satisfies WriteError);\n });\n }\n\n return {\n init(\n dbPath: string,\n cpMode: ChangeProcessorMode,\n pragmas: PragmaConfig,\n logConfig: LogConfig,\n ) {\n lc = createLogContext({log: logConfig}, 'write-worker');\n db = new Database(lc, dbPath);\n applyPragmas(db, pragmas);\n runner = new StatementRunner(db);\n mode = cpMode;\n createProcessor();\n },\n\n getSubscriptionState() {\n return getSubscriptionState(must(runner));\n },\n\n processMessage(downstream: ChangeStreamData) {\n return must(processor).processMessage(must(lc), downstream);\n },\n\n abort() {\n must(processor).abort(must(lc));\n createProcessor();\n },\n\n stop() {\n db?.close();\n db = undefined;\n runner = undefined;\n processor = undefined;\n },\n };\n}\n\nconst api = createAPI();\n\nport.on('message', (msg: Request) => {\n try {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TS can't narrow msg.method + msg.args together\n const result = (api[msg.method] as (...args: any[]) => unknown)(\n ...msg.args,\n );\n // abort is fire-and-forget — no pending slot on the client side.\n if (msg.method !== 'abort') {\n port.postMessage({method: msg.method, result} as Response);\n }\n } catch (e) {\n if (msg.method !== 'abort') {\n port.postMessage({\n method: msg.method,\n error: serializeError(e),\n } as Response);\n }\n }\n});\n"],"mappings":";;;;;;;;;AAsBA,IAAI,CAAC,WACH,OAAM,IAAI,MAAM,8CAA8C;AAGhE,IAAM,OAAO;AAIb,SAAS,YAAiB;CACxB,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CAEJ,SAAS,kBAAkB;AACzB,cAAY,IAAI,gBAAgB,KAAK,OAAO,EAAE,KAAK,KAAK,GAAG,KAAK,QAAQ;AACtE,QAAK,YAAY,EACf,YAAY,eAAe,IAAI,EAChC,CAAsB;IACvB;;AAGJ,QAAO;EACL,KACE,QACA,QACA,SACA,WACA;AACA,QAAK,iBAAiB,EAAC,KAAK,WAAU,EAAE,eAAe;AACvD,QAAK,IAAI,SAAS,IAAI,OAAO;AAC7B,gBAAa,IAAI,QAAQ;AACzB,YAAS,IAAI,gBAAgB,GAAG;AAChC,UAAO;AACP,oBAAiB;;EAGnB,uBAAuB;AACrB,UAAO,qBAAqB,KAAK,OAAO,CAAC;;EAG3C,eAAe,YAA8B;AAC3C,UAAO,KAAK,UAAU,CAAC,eAAe,KAAK,GAAG,EAAE,WAAW;;EAG7D,QAAQ;AACN,QAAK,UAAU,CAAC,MAAM,KAAK,GAAG,CAAC;AAC/B,oBAAiB;;EAGnB,OAAO;AACL,OAAI,OAAO;AACX,QAAK,KAAA;AACL,YAAS,KAAA;AACT,eAAY,KAAA;;EAEf;;AAGH,IAAM,MAAM,WAAW;AAEvB,KAAK,GAAG,YAAY,QAAiB;AACnC,KAAI;EAEF,MAAM,SAAU,IAAI,IAAI,QACtB,GAAG,IAAI,KACR;AAED,MAAI,IAAI,WAAW,QACjB,MAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ;GAAO,CAAa;UAErD,GAAG;AACV,MAAI,IAAI,WAAW,QACjB,MAAK,YAAY;GACf,QAAQ,IAAI;GACZ,OAAO,eAAe,EAAE;GACzB,CAAa;;EAGlB"}

package/out/zero-cache/src/services/running-state.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"running-state.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/running-state.ts"],"names":[],"mappings":"~~AAAA~~,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,cAAc,EAAC,MAAM,8BAA8B,CAAC;~~AAG5D~~,eAAO,MAAM,0BAA0B,QAAQ,CAAC;AAEhD,MAAM,MAAM,WAAW,GAAG;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,MAAM,WAAW,UAAU;IACzB,MAAM,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC;AAEtC;;GAEG;AACH,qBAAa,YAAY;;gBAarB,WAAW,EAAE,MAAM,EACnB,WAAW,CAAC,EAAE,WAAW,EACzB,YAAY,oBAAa,EACzB,OAAO,wBAAiB;IA+B1B,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,IAAI,UAAU,WAEb;IAED;;;;;OAKG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,YAAY,CAAC,CAAC,EAAE,UAAU,GAAG,YAAY;IAMzC;;OAEG;IACH,UAAU,CAAC,KAAK,SAAS,OAAO,EAAE,EAChC,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,IAAI,EAC5B,SAAS,EAAE,MAAM,EACjB,GAAG,IAAI,EAAE,KAAK;IAWhB;;;OAGG;IACG,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAItC;;;OAGG;IACH,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI;IAQzC;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAIxB;;;;;;OAMG;IACG,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;~~IAiB1D~~;;;;;;OAMG;IACH,YAAY;CAKb;AAED;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,QAAQ,CAAC,IAAI,wBAAwB;CACtC"}
1	+ {"version":3,"file":"running-state.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/running-state.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,cAAc,EAAC,MAAM,8BAA8B,CAAC;AAI5D,eAAO,MAAM,0BAA0B,QAAQ,CAAC;AAEhD,MAAM,MAAM,WAAW,GAAG;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,MAAM,WAAW,UAAU;IACzB,MAAM,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC;AAEtC;;GAEG;AACH,qBAAa,YAAY;;gBAarB,WAAW,EAAE,MAAM,EACnB,WAAW,CAAC,EAAE,WAAW,EACzB,YAAY,oBAAa,EACzB,OAAO,wBAAiB;IA+B1B,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,IAAI,UAAU,WAEb;IAED;;;;;OAKG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,YAAY,CAAC,CAAC,EAAE,UAAU,GAAG,YAAY;IAMzC;;OAEG;IACH,UAAU,CAAC,KAAK,SAAS,OAAO,EAAE,EAChC,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,IAAI,EAC5B,SAAS,EAAE,MAAM,EACjB,GAAG,IAAI,EAAE,KAAK;IAWhB;;;OAGG;IACG,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAItC;;;OAGG;IACH,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI;IAQzC;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAIxB;;;;;;OAMG;IACG,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAwB1D;;;;;;OAMG;IACH,YAAY;CAKb;AAED;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,QAAQ,CAAC,IAAI,wBAAwB;CACtC"}

package/out/zero-cache/src/services/running-state.js CHANGED Viewed

@@ -1,6 +1,8 @@
 import { AbortError } from "../../../shared/src/abort-error.js";
 import { sleepWithAbort } from "../../../shared/src/sleep.js";
+import { isPostgresError } from "../types/pg.js";
 import { resolver } from "@rocicorp/resolver";
+import { PG_UNDEFINED_TABLE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/running-state.ts
 var DEFAULT_INITIAL_RETRY_DELAY_MS = 25;
 var DEFAULT_MAX_RETRY_DELAY_MS = 1e4;
@@ -102,6 +104,7 @@ var RunningState = class {
 	async backoff(lc, err) {
 		const delay = this.#retryDelay;
 		this.#retryDelay = Math.min(delay * 2, this.#maxRetryDelay);
+		if (isPostgresError(err, PG_UNDEFINED_TABLE)) err = new AbortError("undefined table", { cause: err });
 		if (err instanceof AbortError || err instanceof UnrecoverableError) {
 			this.resetBackoff();
 			this.stop(lc, err);

package/out/zero-cache/src/services/running-state.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"running-state.js","names":["#serviceName","#controller","#sleep","#setTimeout","#stopped","#initialRetryDelay","#maxRetryDelay","#pendingTimeouts","#retryDelay"],"sources":["../../../../../zero-cache/src/services/running-state.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {sleepWithAbort} from '../../../shared/src/sleep.ts';\n\nconst DEFAULT_INITIAL_RETRY_DELAY_MS = 25;\nexport const DEFAULT_MAX_RETRY_DELAY_MS = 10000;\n\nexport type RetryConfig = {\n initialRetryDelay?: number;\n maxRetryDelay?: number;\n};\n\nexport interface Cancelable {\n cancel(): void;\n}\n\nexport type UnregisterFn = () => void;\n\n/*\n Facilitates lifecycle control with exponential backoff.\n /\nexport class RunningState {\n readonly #serviceName: string;\n readonly #controller: AbortController;\n readonly #sleep: typeof sleepWithAbort;\n readonly #setTimeout: typeof setTimeout;\n readonly #stopped: Promise<void>;\n\n readonly #initialRetryDelay: number;\n readonly #maxRetryDelay: number;\n readonly #pendingTimeouts = new Set<NodeJS.Timeout>();\n #retryDelay: number;\n\n constructor(\n serviceName: string,\n retryConfig?: RetryConfig,\n setTimeoutFn = setTimeout,\n sleeper = sleepWithAbort,\n ) {\n const {\n initialRetryDelay = DEFAULT_INITIAL_RETRY_DELAY_MS,\n maxRetryDelay = DEFAULT_MAX_RETRY_DELAY_MS,\n } = retryConfig ?? {};\n\n this.#serviceName = serviceName;\n this.#initialRetryDelay = initialRetryDelay;\n this.#maxRetryDelay = maxRetryDelay;\n this.#retryDelay = initialRetryDelay;\n\n this.#controller = new AbortController();\n this.#sleep = sleeper;\n this.#setTimeout = setTimeoutFn;\n\n const {promise, resolve} = resolver();\n this.#stopped = promise;\n this.#controller.signal.addEventListener(\n 'abort',\n () => {\n resolve();\n for (const timeout of this.#pendingTimeouts) {\n clearTimeout(timeout);\n }\n this.#pendingTimeouts.clear();\n },\n {once: true},\n );\n }\n\n get signal(): AbortSignal {\n return this.#controller.signal;\n }\n\n get retryDelay() {\n return this.#retryDelay;\n }\n\n /\n Returns `true` until {@link stop()} has been called.\n \n This is usually called as part of the service's main loop\n * conditional to determine if the next iteration should execute.\n /\n shouldRun(): boolean {\n return !this.#controller.signal.aborted;\n }\n\n /\n Registers a Cancelable object to be invoked when {@link stop()} is called.\n * Returns a method to unregister the object.\n /\n cancelOnStop(c: Cancelable): UnregisterFn {\n const onStop = () => c.cancel();\n this.#controller.signal.addEventListener('abort', onStop, {once: true});\n return () => this.#controller.signal.removeEventListener('abort', onStop);\n }\n\n /\n Sets a Timeout that is automatically cancelled if the service is cancelled.\n /\n setTimeout<TArgs extends unknown[]>(\n fn: (...args: TArgs) => void,\n timeoutMs: number,\n ...args: TArgs\n ) {\n const timeout = this.#setTimeout(() => {\n clearTimeout(timeout);\n this.#pendingTimeouts.delete(timeout);\n return fn(...args);\n }, timeoutMs);\n\n this.#pendingTimeouts.add(timeout);\n }\n\n /\n Returns a promise that resolves after `ms` milliseconds or when\n * the service is stopped.\n /\n async sleep(ms: number): Promise<void> {\n await Promise.race(this.#sleep(ms, this.#controller.signal));\n }\n\n /\n Called to stop the service. After this is called, {@link shouldRun()}\n * will return `false` and the {@link stopped()} Promise will be resolved.\n /\n stop(lc: LogContext, err?: unknown): void {\n if (this.shouldRun()) {\n const log = !err \|\| err instanceof AbortError ? 'info' : 'error';\n lc[log]?.(`stopping ${this.#serviceName}`, err ?? '');\n this.#controller.abort();\n }\n }\n\n /\n Returns a Promise that resolves when {@link stop()} is called.\n /\n stopped(): Promise<void> {\n return this.#stopped;\n }\n\n /\n Call in response to an error or unexpected termination in the main\n * loop of the service. The returned Promise will resolve after an\n * exponential delay, or once {@link stop()} is called.\n \n If the supplied `err` is an `AbortError`, the service will shut down.\n /\n async backoff(lc: LogContext, err: unknown): Promise<void> {\n const delay = this.#retryDelay;\n this.#retryDelay = Math.min(delay 2, this.#maxRetryDelay);\n\n if (err instanceof AbortError \|\| err instanceof UnrecoverableError) {\n this.resetBackoff();\n this.stop(lc, err);\n } else if (this.shouldRun()) {\n // Use delay-based log level: higher delay means more retries\n const log: 'info' \| 'warn' \| 'error' =\n delay < 1000 ? 'info' : delay < 6500 ? 'warn' : 'error';\n\n lc[log]?.(`retrying ${this.#serviceName} in ${delay} ms`, err);\n await this.sleep(delay);\n }\n }\n\n /*\n When using {@link backoff()}, this method should be called when the\n * implementation receives a healthy signal (e.g. a successful\n * response). This resets the delay used in {@link backoff()}.\n \n @returns The previous backoff delay\n /\n resetBackoff() {\n const prevDelay = this.#retryDelay;\n this.#retryDelay = this.#initialRetryDelay;\n return prevDelay;\n }\n}\n\n/\n Superclass for Errors that should bypass exponential backoff\n * and immediately shut down the server.\n */\nexport class UnrecoverableError extends Error {\n readonly name = 'UnrecoverableError';\n}\n"],"mappings":"~~;;;;AAKA~~,IAAM,iCAAiC;AACvC,IAAa,6BAA6B;;;;AAgB1C,IAAa,eAAb,MAA0B;CACxB;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,mCAA4B,IAAI,KAAqB;CACrD;CAEA,YACE,aACA,aACA,eAAe,YACf,UAAU,gBACV;EACA,MAAM,EACJ,oBAAoB,gCACpB,gBAAgB,+BACd,eAAe,EAAE;AAErB,QAAA,cAAoB;AACpB,QAAA,oBAA0B;AAC1B,QAAA,gBAAsB;AACtB,QAAA,aAAmB;AAEnB,QAAA,aAAmB,IAAI,iBAAiB;AACxC,QAAA,QAAc;AACd,QAAA,aAAmB;EAEnB,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,UAAgB;AAChB,QAAA,WAAiB,OAAO,iBACtB,eACM;AACJ,YAAS;AACT,QAAK,MAAM,WAAW,MAAA,gBACpB,cAAa,QAAQ;AAEvB,SAAA,gBAAsB,OAAO;KAE/B,EAAC,MAAM,MAAK,CACb;;CAGH,IAAI,SAAsB;AACxB,SAAO,MAAA,WAAiB;;CAG1B,IAAI,aAAa;AACf,SAAO,MAAA;;;;;;;;CAST,YAAqB;AACnB,SAAO,CAAC,MAAA,WAAiB,OAAO;;;;;;CAOlC,aAAa,GAA6B;EACxC,MAAM,eAAe,EAAE,QAAQ;AAC/B,QAAA,WAAiB,OAAO,iBAAiB,SAAS,QAAQ,EAAC,MAAM,MAAK,CAAC;AACvE,eAAa,MAAA,WAAiB,OAAO,oBAAoB,SAAS,OAAO;;;;;CAM3E,WACE,IACA,WACA,GAAG,MACH;EACA,MAAM,UAAU,MAAA,iBAAuB;AACrC,gBAAa,QAAQ;AACrB,SAAA,gBAAsB,OAAO,QAAQ;AACrC,UAAO,GAAG,GAAG,KAAK;KACjB,UAAU;AAEb,QAAA,gBAAsB,IAAI,QAAQ;;;;;;CAOpC,MAAM,MAAM,IAA2B;AACrC,QAAM,QAAQ,KAAK,MAAA,MAAY,IAAI,MAAA,WAAiB,OAAO,CAAC;;;;;;CAO9D,KAAK,IAAgB,KAAqB;AACxC,MAAI,KAAK,WAAW,EAAE;AAEpB,MADY,CAAC,OAAO,eAAe,aAAa,SAAS,WAC/C,YAAY,MAAA,eAAqB,OAAO,GAAG;AACrD,SAAA,WAAiB,OAAO;;;;;;CAO5B,UAAyB;AACvB,SAAO,MAAA;;;;;;;;;CAUT,MAAM,QAAQ,IAAgB,KAA6B;EACzD,MAAM,QAAQ,MAAA;AACd,QAAA,aAAmB,KAAK,IAAI,QAAQ,GAAG,MAAA,cAAoB;~~AAE3D~~,MAAI,eAAe,cAAc,eAAe,oBAAoB;AAClE,QAAK,cAAc;AACnB,QAAK,KAAK,IAAI,IAAI;aACT,KAAK,WAAW,EAAE;AAK3B,MAFE,QAAQ,MAAO,SAAS,QAAQ,OAAO,SAAS,WAExC,YAAY,MAAA,YAAkB,MAAM,MAAM,MAAM,IAAI;AAC9D,SAAM,KAAK,MAAM,MAAM;;;;;;;;;;CAW3B,eAAe;EACb,MAAM,YAAY,MAAA;AAClB,QAAA,aAAmB,MAAA;AACnB,SAAO;;;;;;;AAQX,IAAa,qBAAb,cAAwC,MAAM;CAC5C,OAAgB"}
1	+ {"version":3,"file":"running-state.js","names":["#serviceName","#controller","#sleep","#setTimeout","#stopped","#initialRetryDelay","#maxRetryDelay","#pendingTimeouts","#retryDelay"],"sources":["../../../../../zero-cache/src/services/running-state.ts"],"sourcesContent":["import {PG_UNDEFINED_TABLE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {sleepWithAbort} from '../../../shared/src/sleep.ts';\nimport {isPostgresError} from '../types/pg.ts';\n\nconst DEFAULT_INITIAL_RETRY_DELAY_MS = 25;\nexport const DEFAULT_MAX_RETRY_DELAY_MS = 10000;\n\nexport type RetryConfig = {\n initialRetryDelay?: number;\n maxRetryDelay?: number;\n};\n\nexport interface Cancelable {\n cancel(): void;\n}\n\nexport type UnregisterFn = () => void;\n\n/*\n Facilitates lifecycle control with exponential backoff.\n /\nexport class RunningState {\n readonly #serviceName: string;\n readonly #controller: AbortController;\n readonly #sleep: typeof sleepWithAbort;\n readonly #setTimeout: typeof setTimeout;\n readonly #stopped: Promise<void>;\n\n readonly #initialRetryDelay: number;\n readonly #maxRetryDelay: number;\n readonly #pendingTimeouts = new Set<NodeJS.Timeout>();\n #retryDelay: number;\n\n constructor(\n serviceName: string,\n retryConfig?: RetryConfig,\n setTimeoutFn = setTimeout,\n sleeper = sleepWithAbort,\n ) {\n const {\n initialRetryDelay = DEFAULT_INITIAL_RETRY_DELAY_MS,\n maxRetryDelay = DEFAULT_MAX_RETRY_DELAY_MS,\n } = retryConfig ?? {};\n\n this.#serviceName = serviceName;\n this.#initialRetryDelay = initialRetryDelay;\n this.#maxRetryDelay = maxRetryDelay;\n this.#retryDelay = initialRetryDelay;\n\n this.#controller = new AbortController();\n this.#sleep = sleeper;\n this.#setTimeout = setTimeoutFn;\n\n const {promise, resolve} = resolver();\n this.#stopped = promise;\n this.#controller.signal.addEventListener(\n 'abort',\n () => {\n resolve();\n for (const timeout of this.#pendingTimeouts) {\n clearTimeout(timeout);\n }\n this.#pendingTimeouts.clear();\n },\n {once: true},\n );\n }\n\n get signal(): AbortSignal {\n return this.#controller.signal;\n }\n\n get retryDelay() {\n return this.#retryDelay;\n }\n\n /\n Returns `true` until {@link stop()} has been called.\n \n This is usually called as part of the service's main loop\n * conditional to determine if the next iteration should execute.\n /\n shouldRun(): boolean {\n return !this.#controller.signal.aborted;\n }\n\n /\n Registers a Cancelable object to be invoked when {@link stop()} is called.\n * Returns a method to unregister the object.\n /\n cancelOnStop(c: Cancelable): UnregisterFn {\n const onStop = () => c.cancel();\n this.#controller.signal.addEventListener('abort', onStop, {once: true});\n return () => this.#controller.signal.removeEventListener('abort', onStop);\n }\n\n /\n Sets a Timeout that is automatically cancelled if the service is cancelled.\n /\n setTimeout<TArgs extends unknown[]>(\n fn: (...args: TArgs) => void,\n timeoutMs: number,\n ...args: TArgs\n ) {\n const timeout = this.#setTimeout(() => {\n clearTimeout(timeout);\n this.#pendingTimeouts.delete(timeout);\n return fn(...args);\n }, timeoutMs);\n\n this.#pendingTimeouts.add(timeout);\n }\n\n /\n Returns a promise that resolves after `ms` milliseconds or when\n * the service is stopped.\n /\n async sleep(ms: number): Promise<void> {\n await Promise.race(this.#sleep(ms, this.#controller.signal));\n }\n\n /\n Called to stop the service. After this is called, {@link shouldRun()}\n * will return `false` and the {@link stopped()} Promise will be resolved.\n /\n stop(lc: LogContext, err?: unknown): void {\n if (this.shouldRun()) {\n const log = !err \|\| err instanceof AbortError ? 'info' : 'error';\n lc[log]?.(`stopping ${this.#serviceName}`, err ?? '');\n this.#controller.abort();\n }\n }\n\n /\n Returns a Promise that resolves when {@link stop()} is called.\n /\n stopped(): Promise<void> {\n return this.#stopped;\n }\n\n /\n Call in response to an error or unexpected termination in the main\n * loop of the service. The returned Promise will resolve after an\n * exponential delay, or once {@link stop()} is called.\n \n If the supplied `err` is an `AbortError`, the service will shut down.\n /\n async backoff(lc: LogContext, err: unknown): Promise<void> {\n const delay = this.#retryDelay;\n this.#retryDelay = Math.min(delay 2, this.#maxRetryDelay);\n\n // Common failure mode when the user deletes a pg schema out from under\n // us after init. Turn into an AbortError to force restart, which will\n // restart and re-init the schema.\n if (isPostgresError(err, PG_UNDEFINED_TABLE)) {\n err = new AbortError('undefined table', {cause: err});\n }\n\n if (err instanceof AbortError \|\| err instanceof UnrecoverableError) {\n this.resetBackoff();\n this.stop(lc, err);\n } else if (this.shouldRun()) {\n // Use delay-based log level: higher delay means more retries\n const log: 'info' \| 'warn' \| 'error' =\n delay < 1000 ? 'info' : delay < 6500 ? 'warn' : 'error';\n\n lc[log]?.(`retrying ${this.#serviceName} in ${delay} ms`, err);\n await this.sleep(delay);\n }\n }\n\n /*\n When using {@link backoff()}, this method should be called when the\n * implementation receives a healthy signal (e.g. a successful\n * response). This resets the delay used in {@link backoff()}.\n \n @returns The previous backoff delay\n /\n resetBackoff() {\n const prevDelay = this.#retryDelay;\n this.#retryDelay = this.#initialRetryDelay;\n return prevDelay;\n }\n}\n\n/\n Superclass for Errors that should bypass exponential backoff\n * and immediately shut down the server.\n */\nexport class UnrecoverableError extends Error {\n readonly name = 'UnrecoverableError';\n}\n"],"mappings":";;;;;;AAOA,IAAM,iCAAiC;AACvC,IAAa,6BAA6B;;;;AAgB1C,IAAa,eAAb,MAA0B;CACxB;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,mCAA4B,IAAI,KAAqB;CACrD;CAEA,YACE,aACA,aACA,eAAe,YACf,UAAU,gBACV;EACA,MAAM,EACJ,oBAAoB,gCACpB,gBAAgB,+BACd,eAAe,EAAE;AAErB,QAAA,cAAoB;AACpB,QAAA,oBAA0B;AAC1B,QAAA,gBAAsB;AACtB,QAAA,aAAmB;AAEnB,QAAA,aAAmB,IAAI,iBAAiB;AACxC,QAAA,QAAc;AACd,QAAA,aAAmB;EAEnB,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,UAAgB;AAChB,QAAA,WAAiB,OAAO,iBACtB,eACM;AACJ,YAAS;AACT,QAAK,MAAM,WAAW,MAAA,gBACpB,cAAa,QAAQ;AAEvB,SAAA,gBAAsB,OAAO;KAE/B,EAAC,MAAM,MAAK,CACb;;CAGH,IAAI,SAAsB;AACxB,SAAO,MAAA,WAAiB;;CAG1B,IAAI,aAAa;AACf,SAAO,MAAA;;;;;;;;CAST,YAAqB;AACnB,SAAO,CAAC,MAAA,WAAiB,OAAO;;;;;;CAOlC,aAAa,GAA6B;EACxC,MAAM,eAAe,EAAE,QAAQ;AAC/B,QAAA,WAAiB,OAAO,iBAAiB,SAAS,QAAQ,EAAC,MAAM,MAAK,CAAC;AACvE,eAAa,MAAA,WAAiB,OAAO,oBAAoB,SAAS,OAAO;;;;;CAM3E,WACE,IACA,WACA,GAAG,MACH;EACA,MAAM,UAAU,MAAA,iBAAuB;AACrC,gBAAa,QAAQ;AACrB,SAAA,gBAAsB,OAAO,QAAQ;AACrC,UAAO,GAAG,GAAG,KAAK;KACjB,UAAU;AAEb,QAAA,gBAAsB,IAAI,QAAQ;;;;;;CAOpC,MAAM,MAAM,IAA2B;AACrC,QAAM,QAAQ,KAAK,MAAA,MAAY,IAAI,MAAA,WAAiB,OAAO,CAAC;;;;;;CAO9D,KAAK,IAAgB,KAAqB;AACxC,MAAI,KAAK,WAAW,EAAE;AAEpB,MADY,CAAC,OAAO,eAAe,aAAa,SAAS,WAC/C,YAAY,MAAA,eAAqB,OAAO,GAAG;AACrD,SAAA,WAAiB,OAAO;;;;;;CAO5B,UAAyB;AACvB,SAAO,MAAA;;;;;;;;;CAUT,MAAM,QAAQ,IAAgB,KAA6B;EACzD,MAAM,QAAQ,MAAA;AACd,QAAA,aAAmB,KAAK,IAAI,QAAQ,GAAG,MAAA,cAAoB;AAK3D,MAAI,gBAAgB,KAAK,mBAAmB,CAC1C,OAAM,IAAI,WAAW,mBAAmB,EAAC,OAAO,KAAI,CAAC;AAGvD,MAAI,eAAe,cAAc,eAAe,oBAAoB;AAClE,QAAK,cAAc;AACnB,QAAK,KAAK,IAAI,IAAI;aACT,KAAK,WAAW,EAAE;AAK3B,MAFE,QAAQ,MAAO,SAAS,QAAQ,OAAO,SAAS,WAExC,YAAY,MAAA,YAAkB,MAAM,MAAM,MAAM,IAAI;AAC9D,SAAM,KAAK,MAAM,MAAM;;;;;;;;;;CAW3B,eAAe;EACb,MAAM,YAAY,MAAA;AAClB,QAAA,aAAmB,MAAA;AACnB,SAAO;;;;;;;AAQX,IAAa,qBAAb,cAAwC,MAAM;CAC5C,OAAgB"}

package/out/zero-cache/src/services/view-syncer/cvr-purger.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"cvr-purger.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-purger.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAQ3C,KAAK,OAAO,GAAG;IACb,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,qBAAa,SAAU,YAAW,OAAO;;IACvC,QAAQ,CAAC,EAAE,YAAY;gBAYrB,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAC,qBAAqB,EAAE,gBAAgB,EAAE,iBAAiB,EAAC,EAAE,OAAO,EACrE,uBAAuB,SAA4B;IAc/C,GAAG;~~IAgDT~~,iBAAiB,CACf,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAC,CAAC;IAkF/C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAItB"}
1	+ {"version":3,"file":"cvr-purger.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-purger.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAQ3C,KAAK,OAAO,GAAG;IACb,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,qBAAa,SAAU,YAAW,OAAO;;IACvC,QAAQ,CAAC,EAAE,YAAY;gBAYrB,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAC,qBAAqB,EAAE,gBAAgB,EAAE,iBAAiB,EAAC,EAAE,OAAO,EACrE,uBAAuB,SAA4B;IAc/C,GAAG;IAiDT,iBAAiB,CACf,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAC,CAAC;IAkF/C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAItB"}

package/out/zero-cache/src/services/view-syncer/cvr-purger.js CHANGED Viewed

@@ -37,6 +37,7 @@ var CVRPurger = class {
 		while (this.#state.shouldRun()) try {
 			const start = performance.now();
 			const { purged, remaining } = await this.purgeInactiveCVRs(maxCVRsPerPurge);
+			this.#state.resetBackoff();
 			if (purgeable !== void 0 && remaining > purgeable) {
 				maxCVRsPerPurge += this.#initialBatchSize;
 				this.#lc.info?.(`increased CVRs per purge to ${maxCVRsPerPurge}`);
@@ -47,7 +48,7 @@ var CVRPurger = class {
 			this.#lc.info?.(`purged ${purged} inactive CVRs (${elapsed.toFixed(2)} ms). Next purge in ${purgeInterval} ms`);
 			await this.#state.sleep(purgeInterval);
 		} catch (e) {
-			this.#lc.warn?.(`error encountered while garbage collecting CVRs`, e);
+			await this.#state.backoff(this.#lc, e);
 		}
 	}
 	purgeInactiveCVRs(maxCVRs) {

package/out/zero-cache/src/services/view-syncer/cvr-purger.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"cvr-purger.js","names":["#lc","#db","#schema","#inactivityThresholdMs","#tombstonePurgeThresholdMs","#initialBatchSize","#initialIntervalMs","#state"],"sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-purger.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {READ_COMMITTED} from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {type PostgresDB} from '../../types/pg.ts';\nimport {cvrSchema, type ShardID} from '../../types/shards.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\n\nconst MINUTE = 60 * 1000;\nconst MAX_PURGE_INTERVAL_MS = 16 * MINUTE;\n\n// Purge tombstones after 31 days to facilitate up to a 30-day actives metric.\nconst TOMBSTONE_PURGE_THRESHOLD = 31 * 24 * 60 * 60 * 1000;\n\ntype Options = {\n inactivityThresholdMs: number;\n initialBatchSize: number;\n initialIntervalMs: number;\n};\n\nexport class CVRPurger implements Service {\n readonly id = 'reaper';\n\n readonly #lc: LogContext;\n readonly #db: PostgresDB;\n readonly #schema: string;\n readonly #inactivityThresholdMs: number;\n readonly #tombstonePurgeThresholdMs: number;\n readonly #initialBatchSize: number;\n readonly #initialIntervalMs: number;\n readonly #state = new RunningState('reaper');\n\n constructor(\n lc: LogContext,\n db: PostgresDB,\n shard: ShardID,\n {inactivityThresholdMs, initialBatchSize, initialIntervalMs}: Options,\n tombstonePurgeThreshold = TOMBSTONE_PURGE_THRESHOLD,\n ) {\n this.#lc = lc;\n this.#db = db;\n this.#schema = cvrSchema(shard);\n this.#inactivityThresholdMs = inactivityThresholdMs;\n this.#tombstonePurgeThresholdMs = Math.max(\n tombstonePurgeThreshold,\n inactivityThresholdMs,\n );\n this.#initialBatchSize = initialBatchSize;\n this.#initialIntervalMs = initialIntervalMs;\n }\n\n async run() {\n let purgeable: number \| undefined;\n let maxCVRsPerPurge = this.#initialBatchSize;\n let purgeInterval = this.#initialIntervalMs;\n\n if (this.#initialBatchSize === 0) {\n this.#lc.warn?.(\n `CVR garbage collection is disabled (initialBatchSize = 0)`,\n );\n // Do nothing and just wait to be stopped.\n await this.#state.stopped();\n } else {\n this.#lc.info?.(\n `running cvr-purger with`,\n await this.#db`SHOW statement_timeout`,\n );\n }\n\n while (this.#state.shouldRun()) {\n try {\n const start = performance.now();\n const {purged, remaining} =\n await this.purgeInactiveCVRs(maxCVRsPerPurge);\n\n if (purgeable !== undefined && remaining > purgeable) {\n // If the number of purgeable CVRs has grown even after the purge,\n // increase the number purged per round to achieve a steady state.\n maxCVRsPerPurge += this.#initialBatchSize;\n this.#lc.info?.(`increased CVRs per purge to ${maxCVRsPerPurge}`);\n }\n purgeable = remaining;\n\n purgeInterval =\n purgeable > 0\n ? this.#initialIntervalMs\n : Math.min(purgeInterval * 2, MAX_PURGE_INTERVAL_MS);\n const elapsed = performance.now() - start;\n this.#lc.info?.(\n `purged ${purged} inactive CVRs (${elapsed.toFixed(2)} ms). Next purge in ${purgeInterval} ms`,\n );\n await this.#state.sleep(purgeInterval);\n } catch (e) {\n this.#lc.~~warn?.~~(~~`error~~ ~~encountered while garbage collecting CVRs`,~~ e);\n }\n }\n }\n\n // Exported for testing.\n purgeInactiveCVRs(\n maxCVRs: number,\n ): Promise<{purged: number; remaining: number}> {\n return runTx(\n this.#db,\n async sql => {\n const now = Date.now();\n const threshold = now - this.#inactivityThresholdMs;\n const tombstonePurgeThreshold = now - this.#tombstonePurgeThresholdMs;\n // Implementation note: `FOR UPDATE` will prevent a syncer from\n // concurrently updating the CVR, since the update also performs\n // a `SELECT ... FOR UPDATE`, instead causing that update to\n // fail, which will cause the client to create a new CVR.\n //\n // `SKIP LOCKED` will skip over CVRs that a syncer is already\n // in the process of updating. In this manner, an in-progress\n // update effectively excludes the CVR from the purge.\n const ids = (\n await sql<{clientGroupID: string}[]>`\n SELECT \"clientGroupID\" FROM ${sql(this.#schema)}.instances\n WHERE NOT \"deleted\" AND \"lastActive\" < ${threshold}\n ORDER BY \"lastActive\" ASC\n LIMIT ${maxCVRs}\n FOR UPDATE SKIP LOCKED\n `.values()\n ).flat();\n\n if (ids.length > 0) {\n // Explicitly delete rows from cvr tables from \"bottom\" up. Relying on\n // foreign key cascading deletes can be suboptimal when the foreign key\n // is not a prefix of the primary key (e.g. the \"desires\" foreign key\n // reference to the \"queries\" table is not a prefix of the \"desires\"\n // primary key).\n const stmts = [\n 'desires',\n 'queries',\n 'clients',\n 'rows',\n 'rowsVersion',\n ].map(table =>\n sql`\n DELETE FROM ${sql(this.#schema)}.${sql(table)} \n WHERE \"clientGroupID\" IN ${sql(ids)}`.execute(),\n );\n // Tombstones are written for the `instances` rows, preserving the\n // \"profileID\" and \"lastActive\" columns for computing usage stats.\n //\n // For backwards compatibility (i.e. older zero-caches that do not\n // check the \"deleted\" column) reset the \"version\" to '00' to trigger\n // the ClientNotFound error via\n // view-syncer.ts:checkClientAndCVRVersions()\n stmts.push(\n sql`\n UPDATE ${sql(this.#schema)}.instances\n SET \"deleted\" = TRUE, \n \"version\" = '00', \n \"ttlClock\" = 0,\n \"replicaVersion\" = NULL, \n \"owner\" = NULL,\n \"grantedAt\" = NULL,\n \"clientSchema\" = NULL\n WHERE \"clientGroupID\" IN ${sql(ids)}`.execute(),\n );\n // Tombstone rows are deleted after the tombstonePurgeThreshold.\n stmts.push(\n sql`\n DELETE FROM ${sql(this.#schema)}.instances\n WHERE \"deleted\" AND \"lastActive\" < ${tombstonePurgeThreshold}\n `.execute(),\n );\n await Promise.all(stmts);\n }\n\n const [{remaining}] = await sql<[{remaining: bigint}]>`\n SELECT COUNT(*) AS remaining FROM ${sql(this.#schema)}.instances\n WHERE NOT \"deleted\" AND \"lastActive\" < ${threshold}\n `;\n\n return {purged: ids.length, remaining: Number(remaining)};\n },\n {mode: READ_COMMITTED},\n );\n }\n\n stop(): Promise<void> {\n this.#state.stop(this.#lc);\n return promiseVoid;\n }\n}\n"],"mappings":";;;;;;;AAUA,IAAM,wBAAwB,MADf,KAAK;AAIpB,IAAM,4BAA4B,MAAU,KAAK,KAAK;AAQtD,IAAa,YAAb,MAA0C;CACxC,KAAc;CAEd;CACA;CACA;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,SAAS;CAE5C,YACE,IACA,IACA,OACA,EAAC,uBAAuB,kBAAkB,qBAC1C,0BAA0B,2BAC1B;AACA,QAAA,KAAW;AACX,QAAA,KAAW;AACX,QAAA,SAAe,UAAU,MAAM;AAC/B,QAAA,wBAA8B;AAC9B,QAAA,4BAAkC,KAAK,IACrC,yBACA,sBACD;AACD,QAAA,mBAAyB;AACzB,QAAA,oBAA0B;;CAG5B,MAAM,MAAM;EACV,IAAI;EACJ,IAAI,kBAAkB,MAAA;EACtB,IAAI,gBAAgB,MAAA;AAEpB,MAAI,MAAA,qBAA2B,GAAG;AAChC,SAAA,GAAS,OACP,4DACD;AAED,SAAM,MAAA,MAAY,SAAS;QAE3B,OAAA,GAAS,OACP,2BACA,MAAM,MAAA,EAAQ,yBACf;AAGH,SAAO,MAAA,MAAY,WAAW,CAC5B,KAAI;GACF,MAAM,QAAQ,YAAY,KAAK;GAC/B,MAAM,EAAC,QAAQ,cACb,MAAM,KAAK,kBAAkB,gBAAgB;~~AAE~~/C,OAAI,cAAc,KAAA,KAAa,YAAY,WAAW;AAGpD,uBAAmB,MAAA;AACnB,UAAA,GAAS,OAAO,+BAA+B,kBAAkB;;AAEnE,eAAY;AAEZ,mBACE,YAAY,IACR,MAAA,oBACA,KAAK,IAAI,gBAAgB,GAAG,sBAAsB;GACxD,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,SAAA,GAAS,OACP,UAAU,OAAO,kBAAkB,QAAQ,QAAQ,EAAE,CAAC,sBAAsB,cAAc,KAC3F;AACD,SAAM,MAAA,MAAY,MAAM,cAAc;WAC/B,GAAG;AACV,~~SAAA~~,~~GAAS~~,~~OAAO~~,~~mDAAmD~~,EAAE;;;~~CAM3E~~,kBACE,SAC8C;AAC9C,SAAO,MACL,MAAA,IACA,OAAM,QAAO;GACX,MAAM,MAAM,KAAK,KAAK;GACtB,MAAM,YAAY,MAAM,MAAA;GACxB,MAAM,0BAA0B,MAAM,MAAA;GAStC,MAAM,OACJ,MAAM,GAA8B;wCACN,IAAI,MAAA,OAAa,CAAC;qDACL,UAAU;;oBAE3C,QAAQ;;QAEpB,QAAQ,EACN,MAAM;AAER,OAAI,IAAI,SAAS,GAAG;IAMlB,MAAM,QAAQ;KACZ;KACA;KACA;KACA;KACA;KACD,CAAC,KAAI,UACJ,GAAG;0BACW,IAAI,MAAA,OAAa,CAAC,GAAG,IAAI,MAAM,CAAC;yCACjB,IAAI,IAAI,GAAG,SAAS,CAClD;AAQD,UAAM,KACJ,GAAG;qBACM,IAAI,MAAA,OAAa,CAAC;;;;;;;;yCAQE,IAAI,IAAI,GAAG,SAAS,CAClD;AAED,UAAM,KACJ,GAAG;0BACW,IAAI,MAAA,OAAa,CAAC;mDACO,wBAAwB;YAC/D,SAAS,CACV;AACD,UAAM,QAAQ,IAAI,MAAM;;GAG1B,MAAM,CAAC,EAAC,eAAc,MAAM,GAA0B;4CAClB,IAAI,MAAA,OAAa,CAAC;mDACX,UAAU;;AAGrD,UAAO;IAAC,QAAQ,IAAI;IAAQ,WAAW,OAAO,UAAU;IAAC;KAE3D,EAAC,MAAM,gBAAe,CACvB;;CAGH,OAAsB;AACpB,QAAA,MAAY,KAAK,MAAA,GAAS;AAC1B,SAAO"}
1	+ {"version":3,"file":"cvr-purger.js","names":["#lc","#db","#schema","#inactivityThresholdMs","#tombstonePurgeThresholdMs","#initialBatchSize","#initialIntervalMs","#state"],"sources":["../../../../../../zero-cache/src/services/view-syncer/cvr-purger.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {READ_COMMITTED} from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {type PostgresDB} from '../../types/pg.ts';\nimport {cvrSchema, type ShardID} from '../../types/shards.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\n\nconst MINUTE = 60 * 1000;\nconst MAX_PURGE_INTERVAL_MS = 16 * MINUTE;\n\n// Purge tombstones after 31 days to facilitate up to a 30-day actives metric.\nconst TOMBSTONE_PURGE_THRESHOLD = 31 * 24 * 60 * 60 * 1000;\n\ntype Options = {\n inactivityThresholdMs: number;\n initialBatchSize: number;\n initialIntervalMs: number;\n};\n\nexport class CVRPurger implements Service {\n readonly id = 'reaper';\n\n readonly #lc: LogContext;\n readonly #db: PostgresDB;\n readonly #schema: string;\n readonly #inactivityThresholdMs: number;\n readonly #tombstonePurgeThresholdMs: number;\n readonly #initialBatchSize: number;\n readonly #initialIntervalMs: number;\n readonly #state = new RunningState('reaper');\n\n constructor(\n lc: LogContext,\n db: PostgresDB,\n shard: ShardID,\n {inactivityThresholdMs, initialBatchSize, initialIntervalMs}: Options,\n tombstonePurgeThreshold = TOMBSTONE_PURGE_THRESHOLD,\n ) {\n this.#lc = lc;\n this.#db = db;\n this.#schema = cvrSchema(shard);\n this.#inactivityThresholdMs = inactivityThresholdMs;\n this.#tombstonePurgeThresholdMs = Math.max(\n tombstonePurgeThreshold,\n inactivityThresholdMs,\n );\n this.#initialBatchSize = initialBatchSize;\n this.#initialIntervalMs = initialIntervalMs;\n }\n\n async run() {\n let purgeable: number \| undefined;\n let maxCVRsPerPurge = this.#initialBatchSize;\n let purgeInterval = this.#initialIntervalMs;\n\n if (this.#initialBatchSize === 0) {\n this.#lc.warn?.(\n `CVR garbage collection is disabled (initialBatchSize = 0)`,\n );\n // Do nothing and just wait to be stopped.\n await this.#state.stopped();\n } else {\n this.#lc.info?.(\n `running cvr-purger with`,\n await this.#db`SHOW statement_timeout`,\n );\n }\n\n while (this.#state.shouldRun()) {\n try {\n const start = performance.now();\n const {purged, remaining} =\n await this.purgeInactiveCVRs(maxCVRsPerPurge);\n this.#state.resetBackoff();\n\n if (purgeable !== undefined && remaining > purgeable) {\n // If the number of purgeable CVRs has grown even after the purge,\n // increase the number purged per round to achieve a steady state.\n maxCVRsPerPurge += this.#initialBatchSize;\n this.#lc.info?.(`increased CVRs per purge to ${maxCVRsPerPurge}`);\n }\n purgeable = remaining;\n\n purgeInterval =\n purgeable > 0\n ? this.#initialIntervalMs\n : Math.min(purgeInterval * 2, MAX_PURGE_INTERVAL_MS);\n const elapsed = performance.now() - start;\n this.#lc.info?.(\n `purged ${purged} inactive CVRs (${elapsed.toFixed(2)} ms). Next purge in ${purgeInterval} ms`,\n );\n await this.#state.sleep(purgeInterval);\n } catch (e) {\n await this.#state.backoff(this.#lc, e);\n }\n }\n }\n\n // Exported for testing.\n purgeInactiveCVRs(\n maxCVRs: number,\n ): Promise<{purged: number; remaining: number}> {\n return runTx(\n this.#db,\n async sql => {\n const now = Date.now();\n const threshold = now - this.#inactivityThresholdMs;\n const tombstonePurgeThreshold = now - this.#tombstonePurgeThresholdMs;\n // Implementation note: `FOR UPDATE` will prevent a syncer from\n // concurrently updating the CVR, since the update also performs\n // a `SELECT ... FOR UPDATE`, instead causing that update to\n // fail, which will cause the client to create a new CVR.\n //\n // `SKIP LOCKED` will skip over CVRs that a syncer is already\n // in the process of updating. In this manner, an in-progress\n // update effectively excludes the CVR from the purge.\n const ids = (\n await sql<{clientGroupID: string}[]>`\n SELECT \"clientGroupID\" FROM ${sql(this.#schema)}.instances\n WHERE NOT \"deleted\" AND \"lastActive\" < ${threshold}\n ORDER BY \"lastActive\" ASC\n LIMIT ${maxCVRs}\n FOR UPDATE SKIP LOCKED\n `.values()\n ).flat();\n\n if (ids.length > 0) {\n // Explicitly delete rows from cvr tables from \"bottom\" up. Relying on\n // foreign key cascading deletes can be suboptimal when the foreign key\n // is not a prefix of the primary key (e.g. the \"desires\" foreign key\n // reference to the \"queries\" table is not a prefix of the \"desires\"\n // primary key).\n const stmts = [\n 'desires',\n 'queries',\n 'clients',\n 'rows',\n 'rowsVersion',\n ].map(table =>\n sql`\n DELETE FROM ${sql(this.#schema)}.${sql(table)} \n WHERE \"clientGroupID\" IN ${sql(ids)}`.execute(),\n );\n // Tombstones are written for the `instances` rows, preserving the\n // \"profileID\" and \"lastActive\" columns for computing usage stats.\n //\n // For backwards compatibility (i.e. older zero-caches that do not\n // check the \"deleted\" column) reset the \"version\" to '00' to trigger\n // the ClientNotFound error via\n // view-syncer.ts:checkClientAndCVRVersions()\n stmts.push(\n sql`\n UPDATE ${sql(this.#schema)}.instances\n SET \"deleted\" = TRUE, \n \"version\" = '00', \n \"ttlClock\" = 0,\n \"replicaVersion\" = NULL, \n \"owner\" = NULL,\n \"grantedAt\" = NULL,\n \"clientSchema\" = NULL\n WHERE \"clientGroupID\" IN ${sql(ids)}`.execute(),\n );\n // Tombstone rows are deleted after the tombstonePurgeThreshold.\n stmts.push(\n sql`\n DELETE FROM ${sql(this.#schema)}.instances\n WHERE \"deleted\" AND \"lastActive\" < ${tombstonePurgeThreshold}\n `.execute(),\n );\n await Promise.all(stmts);\n }\n\n const [{remaining}] = await sql<[{remaining: bigint}]>`\n SELECT COUNT(*) AS remaining FROM ${sql(this.#schema)}.instances\n WHERE NOT \"deleted\" AND \"lastActive\" < ${threshold}\n `;\n\n return {purged: ids.length, remaining: Number(remaining)};\n },\n {mode: READ_COMMITTED},\n );\n }\n\n stop(): Promise<void> {\n this.#state.stop(this.#lc);\n return promiseVoid;\n }\n}\n"],"mappings":";;;;;;;AAUA,IAAM,wBAAwB,MADf,KAAK;AAIpB,IAAM,4BAA4B,MAAU,KAAK,KAAK;AAQtD,IAAa,YAAb,MAA0C;CACxC,KAAc;CAEd;CACA;CACA;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,SAAS;CAE5C,YACE,IACA,IACA,OACA,EAAC,uBAAuB,kBAAkB,qBAC1C,0BAA0B,2BAC1B;AACA,QAAA,KAAW;AACX,QAAA,KAAW;AACX,QAAA,SAAe,UAAU,MAAM;AAC/B,QAAA,wBAA8B;AAC9B,QAAA,4BAAkC,KAAK,IACrC,yBACA,sBACD;AACD,QAAA,mBAAyB;AACzB,QAAA,oBAA0B;;CAG5B,MAAM,MAAM;EACV,IAAI;EACJ,IAAI,kBAAkB,MAAA;EACtB,IAAI,gBAAgB,MAAA;AAEpB,MAAI,MAAA,qBAA2B,GAAG;AAChC,SAAA,GAAS,OACP,4DACD;AAED,SAAM,MAAA,MAAY,SAAS;QAE3B,OAAA,GAAS,OACP,2BACA,MAAM,MAAA,EAAQ,yBACf;AAGH,SAAO,MAAA,MAAY,WAAW,CAC5B,KAAI;GACF,MAAM,QAAQ,YAAY,KAAK;GAC/B,MAAM,EAAC,QAAQ,cACb,MAAM,KAAK,kBAAkB,gBAAgB;AAC/C,SAAA,MAAY,cAAc;AAE1B,OAAI,cAAc,KAAA,KAAa,YAAY,WAAW;AAGpD,uBAAmB,MAAA;AACnB,UAAA,GAAS,OAAO,+BAA+B,kBAAkB;;AAEnE,eAAY;AAEZ,mBACE,YAAY,IACR,MAAA,oBACA,KAAK,IAAI,gBAAgB,GAAG,sBAAsB;GACxD,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,SAAA,GAAS,OACP,UAAU,OAAO,kBAAkB,QAAQ,QAAQ,EAAE,CAAC,sBAAsB,cAAc,KAC3F;AACD,SAAM,MAAA,MAAY,MAAM,cAAc;WAC/B,GAAG;AACV,SAAM,MAAA,MAAY,QAAQ,MAAA,IAAU,EAAE;;;CAM5C,kBACE,SAC8C;AAC9C,SAAO,MACL,MAAA,IACA,OAAM,QAAO;GACX,MAAM,MAAM,KAAK,KAAK;GACtB,MAAM,YAAY,MAAM,MAAA;GACxB,MAAM,0BAA0B,MAAM,MAAA;GAStC,MAAM,OACJ,MAAM,GAA8B;wCACN,IAAI,MAAA,OAAa,CAAC;qDACL,UAAU;;oBAE3C,QAAQ;;QAEpB,QAAQ,EACN,MAAM;AAER,OAAI,IAAI,SAAS,GAAG;IAMlB,MAAM,QAAQ;KACZ;KACA;KACA;KACA;KACA;KACD,CAAC,KAAI,UACJ,GAAG;0BACW,IAAI,MAAA,OAAa,CAAC,GAAG,IAAI,MAAM,CAAC;yCACjB,IAAI,IAAI,GAAG,SAAS,CAClD;AAQD,UAAM,KACJ,GAAG;qBACM,IAAI,MAAA,OAAa,CAAC;;;;;;;;yCAQE,IAAI,IAAI,GAAG,SAAS,CAClD;AAED,UAAM,KACJ,GAAG;0BACW,IAAI,MAAA,OAAa,CAAC;mDACO,wBAAwB;YAC/D,SAAS,CACV;AACD,UAAM,QAAQ,IAAI,MAAM;;GAG1B,MAAM,CAAC,EAAC,eAAc,MAAM,GAA0B;4CAClB,IAAI,MAAA,OAAa,CAAC;mDACX,UAAU;;AAGrD,UAAO;IAAC,QAAQ,IAAI;IAAQ,WAAW,OAAO,UAAU;IAAC;KAE3D,EAAC,MAAM,gBAAe,CACvB;;CAGH,OAAsB;AACpB,QAAA,MAAY,KAAK,MAAA,GAAS;AAC1B,SAAO"}