npm - @rocicorp/zero - Versions diffs - 1.2.0-canary.1 → 1.2.0-canary.12 - Mend

@rocicorp/zero 1.2.0-canary.1 → 1.2.0-canary.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (309) hide show

package/out/zero-cache/src/services/replicator/change-processor.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-processor.js","names":["#db","#changeLog","#tableMetadata","#mode","#failService","#tableSpecs","#failure","#currentTx","#fail","#processMessage","#beginTransaction","#lc","#startMs","#version","#jsonFormat","#columnMetadata","#reloadTableSpecs","#tableSpec","#upsert","#getKey","#logSetOp","#logDeleteOp","#delete","#logTruncateOp","#logResetOp","#bumpVersions","#pos","#numChangeLogEntries","#schemaChanged","#completedBackfill"],"sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {DownloadStatus} from '../../../../zero-events/src/status.ts';\nimport {\n createLiteIndexStatement,\n createLiteTableStatement,\n liteColumnDef,\n} from '../../db/create.ts';\nimport {\n computeZqlSpecs,\n listIndexes,\n listTables,\n type LiteTableSpecWithReplicationStatus,\n} from '../../db/lite-tables.ts';\nimport {\n mapPostgresToLite,\n mapPostgresToLiteColumn,\n mapPostgresToLiteIndex,\n} from '../../db/pg-to-lite.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport type {LexiVersion} from '../../types/lexi-version.ts';\nimport {\n JSON_PARSED,\n liteRow,\n type JSONFormat,\n type LiteRow,\n type LiteRowKey,\n type LiteValueType,\n} from '../../types/lite.ts';\nimport {liteTableName} from '../../types/names.ts';\nimport {id} from '../../types/sql.ts';\nimport type {\n BackfillCompleted,\n Change,\n ColumnAdd,\n ColumnDrop,\n ColumnUpdate,\n Identifier,\n IndexCreate,\n IndexDrop,\n MessageBackfill,\n MessageCommit,\n MessageDelete,\n MessageInsert,\n MessageRelation,\n MessageTruncate,\n MessageUpdate,\n TableCreate,\n TableDrop,\n TableRename,\n TableUpdateMetadata,\n} from '../change-source/protocol/current/data.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from './replicator.ts';\nimport {ChangeLog, DEL_OP, SET_OP} from './schema/change-log.ts';\nimport {ColumnMetadataStore} from './schema/column-metadata.ts';\nimport {\n ZERO_VERSION_COLUMN_NAME,\n updateReplicationWatermark,\n} from './schema/replication-state.ts';\nimport {TableMetadataTracker} from './schema/table-metadata.ts';\n\nexport type ChangeProcessorMode = ReplicatorMode \| 'initial-sync';\n\nexport type CommitResult = {\n watermark: string;\n completedBackfill: DownloadStatus \| undefined;\n schemaUpdated: boolean;\n changeLogUpdated: boolean;\n};\n\n/*\n The ChangeProcessor partitions the stream of messages into transactions\n * by creating a {@link TransactionProcessor} when a transaction begins, and dispatching\n * messages to it until the commit is received.\n \n From https://www.postgresql.org/docs/current/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW :\n \n \"The logical replication protocol sends individual transactions one by one.\n * This means that all messages between a pair of Begin and Commit messages\n * belong to the same transaction.\"\n /\nexport class ChangeProcessor {\n readonly #db: StatementRunner;\n readonly #changeLog: ChangeLog;\n readonly #tableMetadata: TableMetadataTracker;\n readonly #mode: ChangeProcessorMode;\n readonly #failService: (lc: LogContext, err: unknown) => void;\n\n // The TransactionProcessor lazily loads table specs into this Map,\n // and reloads them after a schema change. It is cached here to avoid\n // reading them from the DB on every transaction.\n readonly #tableSpecs = new Map<string, LiteTableSpecWithReplicationStatus>();\n\n #currentTx: TransactionProcessor \| null = null;\n\n #failure: Error \| undefined;\n\n constructor(\n db: StatementRunner,\n mode: ChangeProcessorMode,\n failService: (lc: LogContext, err: unknown) => void,\n ) {\n this.#db = db;\n this.#changeLog = new ChangeLog(db.db);\n this.#tableMetadata = new TableMetadataTracker(db.db);\n this.#mode = mode;\n this.#failService = failService;\n }\n\n #fail(lc: LogContext, err: unknown) {\n if (!this.#failure) {\n this.#currentTx?.abort(lc); // roll back any pending transaction.\n\n this.#failure = ensureError(err);\n\n if (!(err instanceof AbortError)) {\n // Propagate the failure up to the service.\n lc.error?.('Message Processing failed:', this.#failure);\n this.#failService(lc, this.#failure);\n }\n }\n }\n\n abort(lc: LogContext) {\n this.#fail(lc, new AbortError());\n }\n\n /* @return If a transaction was committed. /\n processMessage(\n lc: LogContext,\n downstream: ChangeStreamData,\n ): CommitResult \| null {\n const [type, message] = downstream;\n if (this.#failure) {\n lc.debug?.(`Dropping ${message.tag}`);\n return null;\n }\n try {\n const watermark =\n type === 'begin'\n ? downstream[2].commitWatermark\n : type === 'commit'\n ? downstream[2].watermark\n : undefined;\n return this.#processMessage(lc, message, watermark);\n } catch (e) {\n this.#fail(lc, e);\n }\n return null;\n }\n\n #beginTransaction(\n lc: LogContext,\n commitVersion: string,\n jsonFormat: JSONFormat,\n ): TransactionProcessor {\n const start = Date.now();\n\n // litestream can technically hold the lock for an arbitrary amount of time\n // when checkpointing a large commit. Crashing on the busy-timeout in this\n // scenario will either produce a corrupt backup or otherwise prevent\n // replication from proceeding.\n //\n // Instead, retry the lock acquisition indefinitely. If this masks\n // an unknown deadlock situation, manual intervention will be necessary.\n for (let i = 0; ; i++) {\n try {\n return new TransactionProcessor(\n lc,\n this.#db,\n this.#mode,\n this.#changeLog,\n this.#tableMetadata,\n this.#tableSpecs,\n commitVersion,\n jsonFormat,\n );\n } catch (e) {\n if (e instanceof SqliteError && e.code === 'SQLITE_BUSY') {\n lc.warn?.(\n `SQLITE_BUSY for ${Date.now() - start} ms (attempt ${i + 1}). ` +\n `This is only expected if litestream is performing a large ` +\n `checkpoint.`,\n e,\n );\n continue;\n }\n throw e;\n }\n }\n }\n\n /* @return If a transaction was committed. /\n #processMessage(\n lc: LogContext,\n msg: Change,\n watermark: string \| undefined,\n ): CommitResult \| null {\n if (msg.tag === 'begin') {\n if (this.#currentTx) {\n throw new Error(`Already in a transaction ${stringify(msg)}`);\n }\n this.#currentTx = this.#beginTransaction(\n lc,\n must(watermark),\n msg.json ?? JSON_PARSED,\n );\n return null;\n }\n\n // For non-begin messages, there should be a #currentTx set.\n const tx = this.#currentTx;\n if (!tx) {\n throw new Error(\n `Received message outside of transaction: ${stringify(msg)}`,\n );\n }\n\n if (msg.tag === 'commit') {\n // Undef this.#currentTx to allow the assembly of the next transaction.\n this.#currentTx = null;\n\n assert(watermark, 'watermark is required for commit messages');\n return tx.processCommit(msg, watermark);\n }\n\n if (msg.tag === 'rollback') {\n this.#currentTx?.abort(lc);\n this.#currentTx = null;\n return null;\n }\n\n switch (msg.tag) {\n case 'insert':\n tx.processInsert(msg);\n break;\n case 'update':\n tx.processUpdate(msg);\n break;\n case 'delete':\n tx.processDelete(msg);\n break;\n case 'truncate':\n tx.processTruncate(msg);\n break;\n case 'create-table':\n tx.processCreateTable(msg);\n break;\n case 'rename-table':\n tx.processRenameTable(msg);\n break;\n case 'update-table-metadata':\n tx.processTableMetadata(msg);\n break;\n case 'add-column':\n tx.processAddColumn(msg);\n break;\n case 'update-column':\n tx.processUpdateColumn(msg);\n break;\n case 'drop-column':\n tx.processDropColumn(msg);\n break;\n case 'drop-table':\n tx.processDropTable(msg);\n break;\n case 'create-index':\n tx.processCreateIndex(msg);\n break;\n case 'drop-index':\n tx.processDropIndex(msg);\n break;\n case 'backfill':\n tx.processBackfill(msg);\n break;\n case 'backfill-completed':\n tx.processBackfillCompleted(msg);\n break;\n default:\n unreachable(msg);\n }\n\n return null;\n }\n}\n\n/\n The {@link TransactionProcessor} handles the sequence of messages from\n * upstream, from `BEGIN` to `COMMIT` and executes the corresponding mutations\n * on the {@link postgres.TransactionSql} on the replica.\n \n When applying row contents to the replica, the `_0_version` column is added / updated,\n * and a corresponding entry in the `ChangeLog` is added. The version value is derived\n * from the watermark of the preceding transaction (stored as the `nextStateVersion` in the\n * `ReplicationState` table).\n \n Side note: For non-streaming Postgres transactions, the commitEndLsn (and thus\n * commit watermark) is available in the `begin` message, so it could theoretically\n * be used for the row version of changes within the transaction. However, the\n * commitEndLsn is not available in the streaming (in-progress) transaction\n * protocol, and may not be available for CDC streams of other upstream types.\n * Therefore, the zero replication protocol is designed to not require the commit\n * watermark when a transaction begins.\n \n Also of interest is the fact that all INSERT Messages are logically applied as\n * UPSERTs. See {@link processInsert} for the underlying motivation.\n /\nclass TransactionProcessor {\n readonly #lc: LogContext;\n readonly #startMs: number;\n readonly #db: StatementRunner;\n readonly #mode: ChangeProcessorMode;\n readonly #version: LexiVersion;\n readonly #changeLog: ChangeLog;\n readonly #tableMetadata: TableMetadataTracker;\n readonly #tableSpecs: Map<string, LiteTableSpecWithReplicationStatus>;\n readonly #jsonFormat: JSONFormat;\n readonly #columnMetadata: ColumnMetadataStore;\n\n #pos = 0;\n #schemaChanged = false;\n #numChangeLogEntries = 0;\n\n constructor(\n lc: LogContext,\n db: StatementRunner,\n mode: ChangeProcessorMode,\n changeLog: ChangeLog,\n tableMetadata: TableMetadataTracker,\n tableSpecs: Map<string, LiteTableSpecWithReplicationStatus>,\n commitVersion: LexiVersion,\n jsonFormat: JSONFormat,\n ) {\n this.#startMs = Date.now();\n this.#mode = mode;\n this.#jsonFormat = jsonFormat;\n\n switch (mode) {\n case 'serving':\n // Although the Replicator / Incremental Syncer is the only writer of the replica,\n // a `BEGIN CONCURRENT` transaction is used to allow View Syncers to simulate\n // (i.e. and `ROLLBACK`) changes on historic snapshots of the database for the\n // purpose of IVM).\n //\n // This TransactionProcessor is the only logic that will actually\n // `COMMIT` any transactions to the replica.\n db.beginConcurrent();\n break;\n case 'backup':\n // For the backup-replicator (i.e. replication-manager), there are no View Syncers\n // and thus BEGIN CONCURRENT is not necessary. In fact, BEGIN CONCURRENT can cause\n // deadlocks with forced wal-checkpoints (which `litestream replicate` performs),\n // so it is important to use vanilla transactions in this configuration.\n db.beginImmediate();\n break;\n case 'initial-sync':\n // When the ChangeProcessor is used for initial-sync, the calling code\n // handles the transaction boundaries.\n break;\n default:\n unreachable();\n }\n this.#db = db;\n this.#version = commitVersion;\n this.#lc = lc.withContext('version', commitVersion);\n this.#changeLog = changeLog;\n this.#tableMetadata = tableMetadata;\n this.#tableSpecs = tableSpecs;\n // The column_metadata table is guaranteed to exist since the\n // replica-schema.ts migration to v8.\n this.#columnMetadata = must(ColumnMetadataStore.getInstance(db.db));\n\n if (this.#tableSpecs.size === 0) {\n this.#reloadTableSpecs();\n }\n }\n\n #reloadTableSpecs() {\n this.#tableSpecs.clear();\n // zqlSpecs include the primary key derived from unique indexes\n const zqlSpecs = computeZqlSpecs(this.#lc, this.#db.db, {\n includeBackfillingColumns: true,\n });\n for (let spec of listTables(this.#db.db)) {\n if (!spec.primaryKey) {\n spec = {\n ...spec,\n primaryKey: [\n ...(zqlSpecs.get(spec.name)?.tableSpec.primaryKey ?? []),\n ],\n };\n }\n this.#tableSpecs.set(spec.name, spec);\n }\n }\n\n #tableSpec(name: string) {\n return must(this.#tableSpecs.get(name), `Unknown table ${name}`);\n }\n\n #getKey(\n {row, numCols}: {row: LiteRow; numCols: number},\n {relation}: {relation: MessageRelation},\n ): LiteRowKey {\n const keyColumns =\n relation.rowKey.type !== 'full'\n ? relation.rowKey.columns // already a suitable key\n : this.#tableSpec(liteTableName(relation)).primaryKey;\n if (!keyColumns?.length) {\n throw new Error(\n `Cannot replicate table \"${relation.name}\" without a PRIMARY KEY or UNIQUE INDEX`,\n );\n }\n // For the common case (replica identity default), the row is already the\n // key for deletes and updates, in which case a new object can be avoided.\n if (numCols === keyColumns.length) {\n return row;\n }\n const key: Record<string, LiteValueType> = {};\n for (const col of keyColumns) {\n key[col] = row[col];\n }\n return key;\n }\n\n processInsert(insert: MessageInsert) {\n const table = liteTableName(insert.relation);\n const tableSpec = this.#tableSpec(table);\n const newRow = liteRow(insert.new, tableSpec, this.#jsonFormat);\n\n this.#upsert(table, {\n ...newRow.row,\n [ZERO_VERSION_COLUMN_NAME]: this.#version,\n });\n\n if (insert.relation.rowKey.columns.length === 0) {\n // INSERTs can be replicated for rows without a PRIMARY KEY or a\n // UNIQUE INDEX. These are written to the replica but not recorded\n // in the changeLog, because these rows cannot participate in IVM.\n //\n // (Once the table schema has been corrected to include a key, the\n // associated schema change will reset pipelines and data can be\n // loaded via hydration.)\n return;\n }\n const key = this.#getKey(newRow, insert);\n this.#logSetOp(table, key, getBackfilledColumns(newRow.row, tableSpec));\n }\n\n #upsert(table: string, row: LiteRow) {\n const columns = Object.keys(row).map(c => id(c));\n this.#db.run(\n `\n INSERT OR REPLACE INTO ${id(table)} (${columns.join(',')})\n VALUES (${Array.from({length: columns.length}).fill('?').join(',')})\n `,\n Object.values(row),\n );\n }\n\n // Updates by default are applied as UPDATE commands to support partial\n // row specifications from the change source. In particular, this is needed\n // to handle updates for which unchanged TOASTed values are not sent:\n //\n // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html#PROTOCOL-LOGICALREP-MESSAGE-FORMATS-TUPLEDATA\n //\n // However, in certain cases an UPDATE may be received for a row that\n // was not initially synced, such as when, an existing table is added\n // to the app's publication.\n //\n // In order to facilitate \"resumptive\" replication, the logic falls back to\n // an INSERT if the update did not change any rows.\n processUpdate(update: MessageUpdate) {\n const table = liteTableName(update.relation);\n const tableSpec = this.#tableSpec(table);\n const newRow = liteRow(update.new, tableSpec, this.#jsonFormat);\n const row = {...newRow.row, [ZERO_VERSION_COLUMN_NAME]: this.#version};\n\n // update.key is set with the old values if the key has changed.\n const oldKey = update.key\n ? this.#getKey(\n liteRow(update.key, this.#tableSpec(table), this.#jsonFormat),\n update,\n )\n : null;\n const newKey = this.#getKey(newRow, update);\n\n if (oldKey) {\n this.#logDeleteOp(table, oldKey, tableSpec.backfilling);\n }\n this.#logSetOp(table, newKey, getBackfilledColumns(newRow.row, tableSpec));\n\n const currKey = oldKey ?? newKey;\n const conds = Object.keys(currKey).map(col => `${id(col)}=?`);\n const setExprs = Object.keys(row).map(col => `${id(col)}=?`);\n\n const {changes} = this.#db.run(\n `\n UPDATE ${id(table)}\n SET ${setExprs.join(',')}\n WHERE ${conds.join(' AND ')}\n `,\n [...Object.values(row), ...Object.values(currKey)],\n );\n\n // If the UPDATE did not affect any rows, perform an UPSERT of the\n // new row for resumptive replication.\n if (changes === 0) {\n this.#upsert(table, row);\n }\n }\n\n processDelete(del: MessageDelete) {\n const table = liteTableName(del.relation);\n const tableSpec = this.#tableSpec(table);\n const rowKey = this.#getKey(\n liteRow(del.key, tableSpec, this.#jsonFormat),\n del,\n );\n\n this.#delete(table, rowKey);\n this.#logDeleteOp(table, rowKey, tableSpec.backfilling);\n }\n\n #delete(table: string, rowKey: LiteRowKey) {\n const conds = Object.keys(rowKey).map(col => `${id(col)}=?`);\n this.#db.run(\n `DELETE FROM ${id(table)} WHERE ${conds.join(' AND ')}`,\n Object.values(rowKey),\n );\n }\n\n processTruncate(truncate: MessageTruncate) {\n for (const relation of truncate.relations) {\n const table = liteTableName(relation);\n // Update replica data.\n this.#db.run(`DELETE FROM ${id(table)}`);\n\n // Update change log.\n this.#logTruncateOp(table);\n }\n }\n\n processCreateTable(create: TableCreate) {\n if (create.metadata) {\n this.#tableMetadata.setUpstreamMetadata(create.spec, create.metadata);\n }\n const table = mapPostgresToLite(create.spec);\n this.#db.db.exec(createLiteTableStatement(table));\n\n // Write to metadata table\n for (const [colName, colSpec] of Object.entries(create.spec.columns)) {\n this.#columnMetadata.insert(\n table.name,\n colName,\n colSpec,\n create.backfill?.[colName],\n );\n }\n\n if (\n Object.keys(create.backfill ?? {}).length ===\n Object.keys(create.spec.columns).length\n ) {\n this.#reloadTableSpecs();\n } else {\n // Make the table visible immediately unless all of the columns are\n // being backfilled. In the backfill case, the version bump will happen\n // with the backfill is complete.\n this.#logResetOp(table.name);\n }\n this.#lc.info?.(create.tag, table.name);\n }\n\n processTableMetadata(msg: TableUpdateMetadata) {\n this.#tableMetadata.setUpstreamMetadata(msg.table, msg.new);\n }\n\n processRenameTable(rename: TableRename) {\n this.#tableMetadata.rename(rename.old, rename.new);\n\n const oldName = liteTableName(rename.old);\n const newName = liteTableName(rename.new);\n this.#db.db.exec(`ALTER TABLE ${id(oldName)} RENAME TO ${id(newName)}`);\n\n // Rename in metadata table\n this.#columnMetadata.renameTable(oldName, newName);\n\n this.#bumpVersions(rename.new);\n this.#logResetOp(oldName);\n this.#lc.info?.(rename.tag, oldName, newName);\n }\n\n processAddColumn(msg: ColumnAdd) {\n if (msg.tableMetadata) {\n this.#tableMetadata.setUpstreamMetadata(msg.table, msg.tableMetadata);\n }\n const table = liteTableName(msg.table);\n const {name} = msg.column;\n const spec = mapPostgresToLiteColumn(table, msg.column);\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} ADD ${id(name)} ${liteColumnDef(spec)}`,\n );\n\n // Write to metadata table\n this.#columnMetadata.insert(table, name, msg.column.spec, msg.backfill);\n\n if (msg.backfill) {\n this.#reloadTableSpecs();\n } else {\n // Make the new column visible immediately if it's not being backfilled.\n // Otherwise, the version bump will happen with the backfill is complete.\n this.#bumpVersions(msg.table);\n }\n this.#lc.info?.(msg.tag, table, msg.column);\n }\n\n processUpdateColumn(msg: ColumnUpdate) {\n const table = liteTableName(msg.table);\n let oldName = msg.old.name;\n const newName = msg.new.name;\n\n // update-column can ignore defaults because it does not change the values\n // in existing rows.\n //\n // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DESC-SET-DROP-DEFAULT\n //\n // \"The new default value will only apply in subsequent INSERT or UPDATE\n // commands; it does not cause rows already in the table to change.\"\n //\n // This allows support for _changing_ column defaults to any expression,\n // since it does not affect what the replica needs to do.\n const oldSpec = mapPostgresToLiteColumn(table, msg.old, 'ignore-default');\n const newSpec = mapPostgresToLiteColumn(table, msg.new, 'ignore-default');\n\n // The only updates that are relevant are the column name and the data type.\n if (oldName === newName && oldSpec.dataType === newSpec.dataType) {\n this.#lc.info?.(msg.tag, 'no thing to update', oldSpec, newSpec);\n return;\n }\n // If the data type changes, we have to make a new column with the new data type\n // and copy the values over.\n if (oldSpec.dataType !== newSpec.dataType) {\n // Remember (and drop) the indexes that reference the column.\n const indexes = listIndexes(this.#db.db).filter(\n idx => idx.tableName === table && oldName in idx.columns,\n );\n const stmts = indexes.map(idx => `DROP INDEX IF EXISTS ${id(idx.name)};`);\n const tmpName = `tmp.${newName}`;\n stmts.push(`\n ALTER TABLE ${id(table)} ADD ${id(tmpName)} ${liteColumnDef(newSpec)};\n UPDATE ${id(table)} SET ${id(tmpName)} = ${id(oldName)};\n ALTER TABLE ${id(table)} DROP ${id(oldName)};\n `);\n for (const idx of indexes) {\n // Re-create the indexes to reference the new column.\n idx.columns[tmpName] = idx.columns[oldName];\n delete idx.columns[oldName];\n stmts.push(createLiteIndexStatement(idx));\n }\n this.#db.db.exec(stmts.join(''));\n oldName = tmpName;\n }\n if (oldName !== newName) {\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} RENAME ${id(oldName)} TO ${id(newName)}`,\n );\n }\n\n // Update metadata table\n this.#columnMetadata.update(\n table,\n msg.old.name,\n msg.new.name,\n msg.new.spec,\n );\n\n this.#bumpVersions(msg.table);\n this.#lc.info?.(msg.tag, table, msg.new);\n }\n\n processDropColumn(msg: ColumnDrop) {\n const table = liteTableName(msg.table);\n const {column} = msg;\n this.#db.db.exec(`ALTER TABLE ${id(table)} DROP ${id(column)}`);\n\n // Delete from metadata table\n this.#columnMetadata.deleteColumn(table, column);\n\n this.#bumpVersions(msg.table);\n this.#lc.info?.(msg.tag, table, column);\n }\n\n processDropTable(drop: TableDrop) {\n this.#tableMetadata.drop(drop.id);\n\n const name = liteTableName(drop.id);\n this.#db.db.exec(`DROP TABLE IF EXISTS ${id(name)}`);\n\n // Delete from metadata table\n this.#columnMetadata.deleteTable(name);\n\n this.#logResetOp(name);\n this.#lc.info?.(drop.tag, name);\n }\n\n processCreateIndex(create: IndexCreate) {\n const index = mapPostgresToLiteIndex(create.spec);\n this.#db.db.exec(createLiteIndexStatement(index));\n\n // indexes affect tables visibility (e.g. sync-ability is gated on\n // having a unique index), so reset pipelines to refresh table schemas.\n // However, the reset is not necessary if the index is for a table\n // that is not yet visible due to backfilling.\n const tableSpec = must(this.#tableSpecs.get(index.tableName));\n if (\n (tableSpec.backfilling ?? []).length ===\n Object.entries(tableSpec.columns).length - 1 // don't count _0_version\n ) {\n this.#reloadTableSpecs();\n } else {\n this.#logResetOp(index.tableName);\n }\n this.#lc.info?.(create.tag, index.name);\n }\n\n processDropIndex(drop: IndexDrop) {\n const name = liteTableName(drop.id);\n this.#db.db.exec(`DROP INDEX IF EXISTS ${id(name)}`);\n this.#lc.info?.(drop.tag, name);\n }\n\n #bumpVersions(table: Identifier) {\n this.#tableMetadata.setMinRowVersion(table, this.#version);\n this.#logResetOp(liteTableName(table));\n }\n\n /\n @param backfilledColumns `backfilling` columns for which values were set\n /\n #logSetOp(\n table: string,\n key: LiteRowKey,\n backfilledColumns: string[] \| undefined,\n ) {\n // The \"serving\" replicator always writes to the change-log (for IVM).\n // The \"backup\" replicator only needs to write to the change log\n // when writing columns that are being backfilled.\n if (this.#mode === 'serving' \|\| backfilledColumns !== undefined) {\n this.#changeLog.logSetOp(\n this.#version,\n this.#pos++,\n table,\n key,\n backfilledColumns,\n );\n this.#numChangeLogEntries++;\n }\n }\n\n #logDeleteOp(table: string, key: LiteRowKey, backfilling?: string[]) {\n // The \"serving\" replicator always writes to the change-log (for IVM).\n // The \"backup\" replicator only needs to write to the change log\n // when writing columns that are being backfilled.\n if (this.#mode === 'serving' \|\| backfilling?.length) {\n this.#changeLog.logDeleteOp(this.#version, this.#pos++, table, key);\n this.#numChangeLogEntries++;\n }\n }\n\n #logTruncateOp(table: string) {\n if (this.#mode === 'serving') {\n this.#changeLog.logTruncateOp(this.#version, table);\n this.#numChangeLogEntries++;\n }\n }\n\n #logResetOp(table: string) {\n this.#schemaChanged = true;\n if (this.#mode === 'serving') {\n this.#changeLog.logResetOp(this.#version, table);\n this.#numChangeLogEntries++;\n }\n this.#reloadTableSpecs();\n }\n\n processBackfill({relation, watermark, columns, rowValues}: MessageBackfill) {\n const tableName = liteTableName(relation);\n const tableSpec = must(this.#tableSpecs.get(tableName));\n const rowKeyCols = relation.rowKey.columns;\n const cols = [...rowKeyCols, ...columns];\n\n // Common parts of the INSERT sql statement.\n const insertColsStr = [...cols, ZERO_VERSION_COLUMN_NAME].map(id).join(',');\n const qMarks = Array.from({length: cols.length + 1}, () => '?').join(',');\n const rowKeyColsStr = rowKeyCols.map(id).join(',');\n\n let backfilled = 0;\n let skipped = 0;\n for (const v of rowValues) {\n const row = liteRow(\n Object.fromEntries(cols.map((c, i) => [c, v[i]])),\n tableSpec,\n this.#jsonFormat,\n );\n const rowKey = this.#getKey(row, {relation});\n const rowOp = this.#changeLog.getLatestRowOp(tableName, rowKey);\n if (rowOp?.op === DEL_OP && rowOp.stateVersion > watermark) {\n skipped++;\n continue; // the row was deleted after the backfill snapshot\n }\n const updates =\n rowOp?.op === SET_OP\n ? cols.filter(\n c => (rowOp.backfillingColumnVersions[c] ?? '') <= watermark,\n )\n : cols;\n if (updates.length === 0) {\n // row already has newer values for all backfilling columns.\n skipped++;\n continue;\n }\n const updateStmts = updates.map(col => `${id(col)}=excluded.${id(col)}`);\n this.#db.run(\n /sql*/ `\n INSERT INTO ${id(tableName)} (${insertColsStr}) VALUES (${qMarks})\n ON CONFLICT (${rowKeyColsStr})\n DO UPDATE SET ${updateStmts.join(',')};\n `,\n ...Object.values(row.row),\n watermark, // the _0_version for new rows (i.e. table backfill)\n );\n backfilled++;\n }\n\n this.#lc.debug?.(\n `backfilled ${backfilled} rows (skipped ${skipped}) into ${tableName}`,\n );\n }\n\n #completedBackfill: DownloadStatus \| undefined;\n\n processBackfillCompleted({relation, columns, status}: BackfillCompleted) {\n const tableName = liteTableName(relation);\n const rowKeyCols = relation.rowKey.columns;\n const cols = [...rowKeyCols, ...columns];\n\n const columnMetadata = must(ColumnMetadataStore.getInstance(this.#db.db));\n for (const col of cols) {\n columnMetadata.clearBackfilling(tableName, col);\n }\n // Given that new columns are being exposed for every row in the table, bump the\n // row version for all rows.\n this.#bumpVersions(relation);\n if (status) {\n this.#completedBackfill = {table: tableName, columns: cols, ...status};\n }\n this.#lc.info?.(`finished backfilling ${tableName}`);\n\n // Note that there is no need to clear the backfillingColumnVersions values\n // in the changeLog. It could theoretically be done for clarity but:\n // (1) it could be non-trivial in terms of latency introduced and\n // (2) the data must be preserved if _other_ columns are in the process\n // of being backfilled\n //\n // Thus, for speed and simplicity, the values are left as is. (Note that\n // subsequent replicated changes to those rows will clear the values if\n // no backfills are in progress).\n }\n\n processCommit(commit: MessageCommit, watermark: string): CommitResult {\n if (watermark !== this.#version) {\n throw new Error(\n `'commit' version ${watermark} does not match 'begin' version ${\n this.#version\n }: ${stringify(commit)}`,\n );\n }\n updateReplicationWatermark(this.#db, watermark);\n\n if (this.#schemaChanged) {\n const start = Date.now();\n this.#db.db.pragma('optimize');\n this.#lc.info?.(\n `PRAGMA optimized after schema change (${Date.now() - start} ms)`,\n );\n }\n\n if (this.#mode !== 'initial-sync') {\n this.#db.commit();\n }\n\n const elapsedMs = Date.now() - this.#startMs;\n this.#lc.debug?.(`Committed tx@${this.#version} (${elapsedMs} ms)`);\n\n return {\n watermark,\n completedBackfill: this.#completedBackfill,\n schemaUpdated: this.#schemaChanged,\n changeLogUpdated: this.#numChangeLogEntries > 0,\n };\n }\n\n abort(lc: LogContext) {\n lc.info?.(`aborting transaction ${this.#version}`);\n this.#db.rollback();\n }\n}\n\nfunction getBackfilledColumns(\n row: LiteRow,\n {backfilling}: LiteTableSpecWithReplicationStatus,\n): string[] \| undefined {\n if (!backfilling?.length) {\n return undefined; // common case\n }\n return backfilling.filter(col => col in row);\n}\n\nfunction ensureError(err: unknown): Error {\n if (err instanceof Error) {\n return err;\n }\n const error = new Error();\n error.cause = err;\n return error;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsFA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CACA;CAKA,8BAAuB,IAAI,KAAiD;CAE5E,aAA0C;CAE1C;CAEA,YACE,IACA,MACA,aACA;AACA,QAAA,KAAW;AACX,QAAA,YAAkB,IAAI,UAAU,GAAG,GAAG;AACtC,QAAA,gBAAsB,IAAI,qBAAqB,GAAG,GAAG;AACrD,QAAA,OAAa;AACb,QAAA,cAAoB;;CAGtB,MAAM,IAAgB,KAAc;AAClC,MAAI,CAAC,MAAA,SAAe;AAClB,SAAA,WAAiB,MAAM,GAAG;AAE1B,SAAA,UAAgB,YAAY,IAAI;AAEhC,OAAI,EAAE,eAAe,aAAa;AAEhC,OAAG,QAAQ,8BAA8B,MAAA,QAAc;AACvD,UAAA,YAAkB,IAAI,MAAA,QAAc;;;;CAK1C,MAAM,IAAgB;AACpB,QAAA,KAAW,IAAI,IAAI,YAAY,CAAC;;;CAIlC,eACE,IACA,YACqB;EACrB,MAAM,CAAC,MAAM,WAAW;AACxB,MAAI,MAAA,SAAe;AACjB,MAAG,QAAQ,YAAY,QAAQ,MAAM;AACrC,UAAO;;AAET,MAAI;GACF,MAAM,YACJ,SAAS,UACL,WAAW,GAAG,kBACd,SAAS,WACP,WAAW,GAAG,YACd,KAAA;AACR,UAAO,MAAA,eAAqB,IAAI,SAAS,UAAU;WAC5C,GAAG;AACV,SAAA,KAAW,IAAI,EAAE;;AAEnB,SAAO;;CAGT,kBACE,IACA,eACA,YACsB;EACtB,MAAM,QAAQ,KAAK,KAAK;AASxB,OAAK,IAAI,IAAI,IAAK,IAChB,KAAI;AACF,UAAO,IAAI,qBACT,IACA,MAAA,IACA,MAAA,MACA,MAAA,WACA,MAAA,eACA,MAAA,YACA,eACA,WACD;WACM,GAAG;AACV,OAAI,aAAa,eAAe,EAAE,SAAS,eAAe;AACxD,OAAG,OACD,mBAAmB,KAAK,KAAK,GAAG,MAAM,eAAe,IAAI,EAAE,2EAG3D,EACD;AACD;;AAEF,SAAM;;;;CAMZ,gBACE,IACA,KACA,WACqB;AACrB,MAAI,IAAI,QAAQ,SAAS;AACvB,OAAI,MAAA,UACF,OAAM,IAAI,MAAM,4BAA4B,UAAU,IAAI,GAAG;AAE/D,SAAA,YAAkB,MAAA,iBAChB,IACA,KAAK,UAAU,EACf,IAAI,QAAA,IACL;AACD,UAAO;;EAIT,MAAM,KAAK,MAAA;AACX,MAAI,CAAC,GACH,OAAM,IAAI,MACR,4CAA4C,UAAU,IAAI,GAC3D;AAGH,MAAI,IAAI,QAAQ,UAAU;AAExB,SAAA,YAAkB;AAElB,UAAO,WAAW,4CAA4C;AAC9D,UAAO,GAAG,cAAc,KAAK,UAAU;;AAGzC,MAAI,IAAI,QAAQ,YAAY;AAC1B,SAAA,WAAiB,MAAM,GAAG;AAC1B,SAAA,YAAkB;AAClB,UAAO;;AAGT,UAAQ,IAAI,KAAZ;GACE,KAAK;AACH,OAAG,cAAc,IAAI;AACrB;GACF,KAAK;AACH,OAAG,cAAc,IAAI;AACrB;GACF,KAAK;AACH,OAAG,cAAc,IAAI;AACrB;GACF,KAAK;AACH,OAAG,gBAAgB,IAAI;AACvB;GACF,KAAK;AACH,OAAG,mBAAmB,IAAI;AAC1B;GACF,KAAK;AACH,OAAG,mBAAmB,IAAI;AAC1B;GACF,KAAK;AACH,OAAG,qBAAqB,IAAI;AAC5B;GACF,KAAK;AACH,OAAG,iBAAiB,IAAI;AACxB;GACF,KAAK;AACH,OAAG,oBAAoB,IAAI;AAC3B;GACF,KAAK;AACH,OAAG,kBAAkB,IAAI;AACzB;GACF,KAAK;AACH,OAAG,iBAAiB,IAAI;AACxB;GACF,KAAK;AACH,OAAG,mBAAmB,IAAI;AAC1B;GACF,KAAK;AACH,OAAG,iBAAiB,IAAI;AACxB;GACF,KAAK;AACH,OAAG,gBAAgB,IAAI;AACvB;GACF,KAAK;AACH,OAAG,yBAAyB,IAAI;AAChC;GACF,QACE,aAAY,IAAI;;AAGpB,SAAO;;;;;;;;;;;;;;;;;;;;;;;;AAyBX,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,OAAO;CACP,iBAAiB;CACjB,uBAAuB;CAEvB,YACE,IACA,IACA,MACA,WACA,eACA,YACA,eACA,YACA;AACA,QAAA,UAAgB,KAAK,KAAK;AAC1B,QAAA,OAAa;AACb,QAAA,aAAmB;AAEnB,UAAQ,MAAR;GACE,KAAK;AAQH,OAAG,iBAAiB;AACpB;GACF,KAAK;AAKH,OAAG,gBAAgB;AACnB;GACF,KAAK,eAGH;GACF,QACE,cAAa;;AAEjB,QAAA,KAAW;AACX,QAAA,UAAgB;AAChB,QAAA,KAAW,GAAG,YAAY,WAAW,cAAc;AACnD,QAAA,YAAkB;AAClB,QAAA,gBAAsB;AACtB,QAAA,aAAmB;AAGnB,QAAA,iBAAuB,KAAK,oBAAoB,YAAY,GAAG,GAAG,CAAC;AAEnE,MAAI,MAAA,WAAiB,SAAS,EAC5B,OAAA,kBAAwB;;CAI5B,oBAAoB;AAClB,QAAA,WAAiB,OAAO;EAExB,MAAM,WAAW,gBAAgB,MAAA,IAAU,MAAA,GAAS,IAAI,EACtD,2BAA2B,MAC5B,CAAC;AACF,OAAK,IAAI,QAAQ,WAAW,MAAA,GAAS,GAAG,EAAE;AACxC,OAAI,CAAC,KAAK,WACR,QAAO;IACL,GAAG;IACH,YAAY,CACV,GAAI,SAAS,IAAI,KAAK,KAAK,EAAE,UAAU,cAAc,EAAE,CACxD;IACF;AAEH,SAAA,WAAiB,IAAI,KAAK,MAAM,KAAK;;;CAIzC,WAAW,MAAc;AACvB,SAAO,KAAK,MAAA,WAAiB,IAAI,KAAK,EAAE,iBAAiB,OAAO;;CAGlE,QACE,EAAC,KAAK,WACN,EAAC,YACW;EACZ,MAAM,aACJ,SAAS,OAAO,SAAS,SACrB,SAAS,OAAO,UAChB,MAAA,UAAgB,cAAc,SAAS,CAAC,CAAC;AAC/C,MAAI,CAAC,YAAY,OACf,OAAM,IAAI,MACR,2BAA2B,SAAS,KAAK,yCAC1C;AAIH,MAAI,YAAY,WAAW,OACzB,QAAO;EAET,MAAM,MAAqC,EAAE;AAC7C,OAAK,MAAM,OAAO,WAChB,KAAI,OAAO,IAAI;AAEjB,SAAO;;CAGT,cAAc,QAAuB;EACnC,MAAM,QAAQ,cAAc,OAAO,SAAS;EAC5C,MAAM,YAAY,MAAA,UAAgB,MAAM;EACxC,MAAM,SAAS,QAAQ,OAAO,KAAK,WAAW,MAAA,WAAiB;AAE/D,QAAA,OAAa,OAAO;GAClB,GAAG,OAAO;IACT,2BAA2B,MAAA;GAC7B,CAAC;AAEF,MAAI,OAAO,SAAS,OAAO,QAAQ,WAAW,EAQ5C;EAEF,MAAM,MAAM,MAAA,OAAa,QAAQ,OAAO;AACxC,QAAA,SAAe,OAAO,KAAK,qBAAqB,OAAO,KAAK,UAAU,CAAC;;CAGzE,QAAQ,OAAe,KAAc;EACnC,MAAM,UAAU,OAAO,KAAK,IAAI,CAAC,KAAI,MAAK,GAAG,EAAE,CAAC;AAChD,QAAA,GAAS,IACP;+BACyB,GAAG,MAAM,CAAC,IAAI,QAAQ,KAAK,IAAI,CAAC;kBAC7C,MAAM,KAAK,EAAC,QAAQ,QAAQ,QAAO,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC;SAErE,OAAO,OAAO,IAAI,CACnB;;CAeH,cAAc,QAAuB;EACnC,MAAM,QAAQ,cAAc,OAAO,SAAS;EAC5C,MAAM,YAAY,MAAA,UAAgB,MAAM;EACxC,MAAM,SAAS,QAAQ,OAAO,KAAK,WAAW,MAAA,WAAiB;EAC/D,MAAM,MAAM;GAAC,GAAG,OAAO;IAAM,2BAA2B,MAAA;GAAc;EAGtE,MAAM,SAAS,OAAO,MAClB,MAAA,OACE,QAAQ,OAAO,KAAK,MAAA,UAAgB,MAAM,EAAE,MAAA,WAAiB,EAC7D,OACD,GACD;EACJ,MAAM,SAAS,MAAA,OAAa,QAAQ,OAAO;AAE3C,MAAI,OACF,OAAA,YAAkB,OAAO,QAAQ,UAAU,YAAY;AAEzD,QAAA,SAAe,OAAO,QAAQ,qBAAqB,OAAO,KAAK,UAAU,CAAC;EAE1E,MAAM,UAAU,UAAU;EAC1B,MAAM,QAAQ,OAAO,KAAK,QAAQ,CAAC,KAAI,QAAO,GAAG,GAAG,IAAI,CAAC,IAAI;EAC7D,MAAM,WAAW,OAAO,KAAK,IAAI,CAAC,KAAI,QAAO,GAAG,GAAG,IAAI,CAAC,IAAI;EAE5D,MAAM,EAAC,YAAW,MAAA,GAAS,IACzB;eACS,GAAG,MAAM,CAAC;cACX,SAAS,KAAK,IAAI,CAAC;gBACjB,MAAM,KAAK,QAAQ,CAAC;SAE9B,CAAC,GAAG,OAAO,OAAO,IAAI,EAAE,GAAG,OAAO,OAAO,QAAQ,CAAC,CACnD;AAID,MAAI,YAAY,EACd,OAAA,OAAa,OAAO,IAAI;;CAI5B,cAAc,KAAoB;EAChC,MAAM,QAAQ,cAAc,IAAI,SAAS;EACzC,MAAM,YAAY,MAAA,UAAgB,MAAM;EACxC,MAAM,SAAS,MAAA,OACb,QAAQ,IAAI,KAAK,WAAW,MAAA,WAAiB,EAC7C,IACD;AAED,QAAA,OAAa,OAAO,OAAO;AAC3B,QAAA,YAAkB,OAAO,QAAQ,UAAU,YAAY;;CAGzD,QAAQ,OAAe,QAAoB;EACzC,MAAM,QAAQ,OAAO,KAAK,OAAO,CAAC,KAAI,QAAO,GAAG,GAAG,IAAI,CAAC,IAAI;AAC5D,QAAA,GAAS,IACP,eAAe,GAAG,MAAM,CAAC,SAAS,MAAM,KAAK,QAAQ,IACrD,OAAO,OAAO,OAAO,CACtB;;CAGH,gBAAgB,UAA2B;AACzC,OAAK,MAAM,YAAY,SAAS,WAAW;GACzC,MAAM,QAAQ,cAAc,SAAS;AAErC,SAAA,GAAS,IAAI,eAAe,GAAG,MAAM,GAAG;AAGxC,SAAA,cAAoB,MAAM;;;CAI9B,mBAAmB,QAAqB;AACtC,MAAI,OAAO,SACT,OAAA,cAAoB,oBAAoB,OAAO,MAAM,OAAO,SAAS;EAEvE,MAAM,QAAQ,kBAAkB,OAAO,KAAK;AAC5C,QAAA,GAAS,GAAG,KAAK,yBAAyB,MAAM,CAAC;AAGjD,OAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,OAAO,KAAK,QAAQ,CAClE,OAAA,eAAqB,OACnB,MAAM,MACN,SACA,SACA,OAAO,WAAW,SACnB;AAGH,MACE,OAAO,KAAK,OAAO,YAAY,EAAE,CAAC,CAAC,WACnC,OAAO,KAAK,OAAO,KAAK,QAAQ,CAAC,OAEjC,OAAA,kBAAwB;MAKxB,OAAA,WAAiB,MAAM,KAAK;AAE9B,QAAA,GAAS,OAAO,OAAO,KAAK,MAAM,KAAK;;CAGzC,qBAAqB,KAA0B;AAC7C,QAAA,cAAoB,oBAAoB,IAAI,OAAO,IAAI,IAAI;;CAG7D,mBAAmB,QAAqB;AACtC,QAAA,cAAoB,OAAO,OAAO,KAAK,OAAO,IAAI;EAElD,MAAM,UAAU,cAAc,OAAO,IAAI;EACzC,MAAM,UAAU,cAAc,OAAO,IAAI;AACzC,QAAA,GAAS,GAAG,KAAK,eAAe,GAAG,QAAQ,CAAC,aAAa,GAAG,QAAQ,GAAG;AAGvE,QAAA,eAAqB,YAAY,SAAS,QAAQ;AAElD,QAAA,aAAmB,OAAO,IAAI;AAC9B,QAAA,WAAiB,QAAQ;AACzB,QAAA,GAAS,OAAO,OAAO,KAAK,SAAS,QAAQ;;CAG/C,iBAAiB,KAAgB;AAC/B,MAAI,IAAI,cACN,OAAA,cAAoB,oBAAoB,IAAI,OAAO,IAAI,cAAc;EAEvE,MAAM,QAAQ,cAAc,IAAI,MAAM;EACtC,MAAM,EAAC,SAAQ,IAAI;EACnB,MAAM,OAAO,wBAAwB,OAAO,IAAI,OAAO;AACvD,QAAA,GAAS,GAAG,KACV,eAAe,GAAG,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,cAAc,KAAK,GAChE;AAGD,QAAA,eAAqB,OAAO,OAAO,MAAM,IAAI,OAAO,MAAM,IAAI,SAAS;AAEvE,MAAI,IAAI,SACN,OAAA,kBAAwB;MAIxB,OAAA,aAAmB,IAAI,MAAM;AAE/B,QAAA,GAAS,OAAO,IAAI,KAAK,OAAO,IAAI,OAAO;;CAG7C,oBAAoB,KAAmB;EACrC,MAAM,QAAQ,cAAc,IAAI,MAAM;EACtC,IAAI,UAAU,IAAI,IAAI;EACtB,MAAM,UAAU,IAAI,IAAI;EAYxB,MAAM,UAAU,wBAAwB,OAAO,IAAI,KAAK,iBAAiB;EACzE,MAAM,UAAU,wBAAwB,OAAO,IAAI,KAAK,iBAAiB;AAGzE,MAAI,YAAY,WAAW,QAAQ,aAAa,QAAQ,UAAU;AAChE,SAAA,GAAS,OAAO,IAAI,KAAK,sBAAsB,SAAS,QAAQ;AAChE;;AAIF,MAAI,QAAQ,aAAa,QAAQ,UAAU;GAEzC,MAAM,UAAU,YAAY,MAAA,GAAS,GAAG,CAAC,QACvC,QAAO,IAAI,cAAc,SAAS,WAAW,IAAI,QAClD;GACD,MAAM,QAAQ,QAAQ,KAAI,QAAO,wBAAwB,GAAG,IAAI,KAAK,CAAC,GAAG;GACzE,MAAM,UAAU,OAAO;AACvB,SAAM,KAAK;sBACK,GAAG,MAAM,CAAC,OAAO,GAAG,QAAQ,CAAC,GAAG,cAAc,QAAQ,CAAC;iBAC5D,GAAG,MAAM,CAAC,OAAO,GAAG,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC;sBACzC,GAAG,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;UAC1C;AACJ,QAAK,MAAM,OAAO,SAAS;AAEzB,QAAI,QAAQ,WAAW,IAAI,QAAQ;AACnC,WAAO,IAAI,QAAQ;AACnB,UAAM,KAAK,yBAAyB,IAAI,CAAC;;AAE3C,SAAA,GAAS,GAAG,KAAK,MAAM,KAAK,GAAG,CAAC;AAChC,aAAU;;AAEZ,MAAI,YAAY,QACd,OAAA,GAAS,GAAG,KACV,eAAe,GAAG,MAAM,CAAC,UAAU,GAAG,QAAQ,CAAC,MAAM,GAAG,QAAQ,GACjE;AAIH,QAAA,eAAqB,OACnB,OACA,IAAI,IAAI,MACR,IAAI,IAAI,MACR,IAAI,IAAI,KACT;AAED,QAAA,aAAmB,IAAI,MAAM;AAC7B,QAAA,GAAS,OAAO,IAAI,KAAK,OAAO,IAAI,IAAI;;CAG1C,kBAAkB,KAAiB;EACjC,MAAM,QAAQ,cAAc,IAAI,MAAM;EACtC,MAAM,EAAC,WAAU;AACjB,QAAA,GAAS,GAAG,KAAK,eAAe,GAAG,MAAM,CAAC,QAAQ,GAAG,OAAO,GAAG;AAG/D,QAAA,eAAqB,aAAa,OAAO,OAAO;AAEhD,QAAA,aAAmB,IAAI,MAAM;AAC7B,QAAA,GAAS,OAAO,IAAI,KAAK,OAAO,OAAO;;CAGzC,iBAAiB,MAAiB;AAChC,QAAA,cAAoB,KAAK,KAAK,GAAG;EAEjC,MAAM,OAAO,cAAc,KAAK,GAAG;AACnC,QAAA,GAAS,GAAG,KAAK,wBAAwB,GAAG,KAAK,GAAG;AAGpD,QAAA,eAAqB,YAAY,KAAK;AAEtC,QAAA,WAAiB,KAAK;AACtB,QAAA,GAAS,OAAO,KAAK,KAAK,KAAK;;CAGjC,mBAAmB,QAAqB;EACtC,MAAM,QAAQ,uBAAuB,OAAO,KAAK;AACjD,QAAA,GAAS,GAAG,KAAK,yBAAyB,MAAM,CAAC;EAMjD,MAAM,YAAY,KAAK,MAAA,WAAiB,IAAI,MAAM,UAAU,CAAC;AAC7D,OACG,UAAU,eAAe,EAAE,EAAE,WAC9B,OAAO,QAAQ,UAAU,QAAQ,CAAC,SAAS,EAE3C,OAAA,kBAAwB;MAExB,OAAA,WAAiB,MAAM,UAAU;AAEnC,QAAA,GAAS,OAAO,OAAO,KAAK,MAAM,KAAK;;CAGzC,iBAAiB,MAAiB;EAChC,MAAM,OAAO,cAAc,KAAK,GAAG;AACnC,QAAA,GAAS,GAAG,KAAK,wBAAwB,GAAG,KAAK,GAAG;AACpD,QAAA,GAAS,OAAO,KAAK,KAAK,KAAK;;CAGjC,cAAc,OAAmB;AAC/B,QAAA,cAAoB,iBAAiB,OAAO,MAAA,QAAc;AAC1D,QAAA,WAAiB,cAAc,MAAM,CAAC;;;;;CAMxC,UACE,OACA,KACA,mBACA;AAIA,MAAI,MAAA,SAAe,aAAa,sBAAsB,KAAA,GAAW;AAC/D,SAAA,UAAgB,SACd,MAAA,SACA,MAAA,OACA,OACA,KACA,kBACD;AACD,SAAA;;;CAIJ,aAAa,OAAe,KAAiB,aAAwB;AAInE,MAAI,MAAA,SAAe,aAAa,aAAa,QAAQ;AACnD,SAAA,UAAgB,YAAY,MAAA,SAAe,MAAA,OAAa,OAAO,IAAI;AACnE,SAAA;;;CAIJ,eAAe,OAAe;AAC5B,MAAI,MAAA,SAAe,WAAW;AAC5B,SAAA,UAAgB,cAAc,MAAA,SAAe,MAAM;AACnD,SAAA;;;CAIJ,YAAY,OAAe;AACzB,QAAA,gBAAsB;AACtB,MAAI,MAAA,SAAe,WAAW;AAC5B,SAAA,UAAgB,WAAW,MAAA,SAAe,MAAM;AAChD,SAAA;;AAEF,QAAA,kBAAwB;;CAG1B,gBAAgB,EAAC,UAAU,WAAW,SAAS,aAA6B;EAC1E,MAAM,YAAY,cAAc,SAAS;EACzC,MAAM,YAAY,KAAK,MAAA,WAAiB,IAAI,UAAU,CAAC;EACvD,MAAM,aAAa,SAAS,OAAO;EACnC,MAAM,OAAO,CAAC,GAAG,YAAY,GAAG,QAAQ;EAGxC,MAAM,gBAAgB,CAAC,GAAG,MAAM,yBAAyB,CAAC,IAAI,GAAG,CAAC,KAAK,IAAI;EAC3E,MAAM,SAAS,MAAM,KAAK,EAAC,QAAQ,KAAK,SAAS,GAAE,QAAQ,IAAI,CAAC,KAAK,IAAI;EACzE,MAAM,gBAAgB,WAAW,IAAI,GAAG,CAAC,KAAK,IAAI;EAElD,IAAI,aAAa;EACjB,IAAI,UAAU;AACd,OAAK,MAAM,KAAK,WAAW;GACzB,MAAM,MAAM,QACV,OAAO,YAAY,KAAK,KAAK,GAAG,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,EACjD,WACA,MAAA,WACD;GACD,MAAM,SAAS,MAAA,OAAa,KAAK,EAAC,UAAS,CAAC;GAC5C,MAAM,QAAQ,MAAA,UAAgB,eAAe,WAAW,OAAO;AAC/D,OAAI,OAAO,OAAA,OAAiB,MAAM,eAAe,WAAW;AAC1D;AACA;;GAEF,MAAM,UACJ,OAAO,OAAA,MACH,KAAK,QACH,OAAM,MAAM,0BAA0B,MAAM,OAAO,UACpD,GACD;AACN,OAAI,QAAQ,WAAW,GAAG;AAExB;AACA;;GAEF,MAAM,cAAc,QAAQ,KAAI,QAAO,GAAG,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG;AACxE,SAAA,GAAS,IACC;sBACM,GAAG,UAAU,CAAC,IAAI,cAAc,YAAY,OAAO;yBAChD,cAAc;0BACb,YAAY,KAAK,IAAI,CAAC;SAExC,GAAG,OAAO,OAAO,IAAI,IAAI,EACzB,UACD;AACD;;AAGF,QAAA,GAAS,QACP,cAAc,WAAW,iBAAiB,QAAQ,SAAS,YAC5D;;CAGH;CAEA,yBAAyB,EAAC,UAAU,SAAS,UAA4B;EACvE,MAAM,YAAY,cAAc,SAAS;EAEzC,MAAM,OAAO,CAAC,GADK,SAAS,OAAO,SACN,GAAG,QAAQ;EAExC,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,MAAA,GAAS,GAAG,CAAC;AACzE,OAAK,MAAM,OAAO,KAChB,gBAAe,iBAAiB,WAAW,IAAI;AAIjD,QAAA,aAAmB,SAAS;AAC5B,MAAI,OACF,OAAA,oBAA0B;GAAC,OAAO;GAAW,SAAS;GAAM,GAAG;GAAO;AAExE,QAAA,GAAS,OAAO,wBAAwB,YAAY;;CAatD,cAAc,QAAuB,WAAiC;AACpE,MAAI,cAAc,MAAA,QAChB,OAAM,IAAI,MACR,oBAAoB,UAAU,kCAC5B,MAAA,QACD,IAAI,UAAU,OAAO,GACvB;AAEH,6BAA2B,MAAA,IAAU,UAAU;AAE/C,MAAI,MAAA,eAAqB;GACvB,MAAM,QAAQ,KAAK,KAAK;AACxB,SAAA,GAAS,GAAG,OAAO,WAAW;AAC9B,SAAA,GAAS,OACP,yCAAyC,KAAK,KAAK,GAAG,MAAM,MAC7D;;AAGH,MAAI,MAAA,SAAe,eACjB,OAAA,GAAS,QAAQ;EAGnB,MAAM,YAAY,KAAK,KAAK,GAAG,MAAA;AAC/B,QAAA,GAAS,QAAQ,gBAAgB,MAAA,QAAc,IAAI,UAAU,MAAM;AAEnE,SAAO;GACL;GACA,mBAAmB,MAAA;GACnB,eAAe,MAAA;GACf,kBAAkB,MAAA,sBAA4B;GAC/C;;CAGH,MAAM,IAAgB;AACpB,KAAG,OAAO,wBAAwB,MAAA,UAAgB;AAClD,QAAA,GAAS,UAAU;;;AAIvB,SAAS,qBACP,KACA,EAAC,eACqB;AACtB,KAAI,CAAC,aAAa,OAChB;AAEF,QAAO,YAAY,QAAO,QAAO,OAAO,IAAI;;AAG9C,SAAS,YAAY,KAAqB;AACxC,KAAI,eAAe,MACjB,QAAO;CAET,MAAM,wBAAQ,IAAI,OAAO;AACzB,OAAM,QAAQ;AACd,QAAO"}
1	+ {"version":3,"file":"change-processor.js","names":["#db","#changeLog","#tableMetadata","#mode","#failService","#tableSpecs","#failure","#currentTx","#fail","#processMessage","#beginTransaction","#lc","#startMs","#version","#jsonFormat","#columnMetadata","#reloadTableSpecs","#tableSpec","#upsert","#getKey","#logSetOp","#logDeleteOp","#delete","#logTruncateOp","#logResetOp","#bumpVersions","#pos","#numChangeLogEntries","#schemaChanged","#completedBackfill"],"sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {DownloadStatus} from '../../../../zero-events/src/status.ts';\nimport {\n createLiteIndexStatement,\n createLiteTableStatement,\n liteColumnDef,\n} from '../../db/create.ts';\nimport {\n computeZqlSpecs,\n listIndexes,\n listTables,\n type LiteTableSpecWithReplicationStatus,\n} from '../../db/lite-tables.ts';\nimport {\n mapPostgresToLite,\n mapPostgresToLiteColumn,\n mapPostgresToLiteIndex,\n} from '../../db/pg-to-lite.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport type {LexiVersion} from '../../types/lexi-version.ts';\nimport {\n JSON_PARSED,\n liteRow,\n type JSONFormat,\n type LiteRow,\n type LiteRowKey,\n type LiteValueType,\n} from '../../types/lite.ts';\nimport {liteTableName} from '../../types/names.ts';\nimport {id} from '../../types/sql.ts';\nimport type {\n BackfillCompleted,\n Change,\n ColumnAdd,\n ColumnDrop,\n ColumnUpdate,\n Identifier,\n IndexCreate,\n IndexDrop,\n MessageBackfill,\n MessageCommit,\n MessageDelete,\n MessageInsert,\n MessageRelation,\n MessageTruncate,\n MessageUpdate,\n TableCreate,\n TableDrop,\n TableRename,\n TableUpdateMetadata,\n} from '../change-source/protocol/current/data.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from './replicator.ts';\nimport {ChangeLog, DEL_OP, SET_OP} from './schema/change-log.ts';\nimport {ColumnMetadataStore} from './schema/column-metadata.ts';\nimport {\n ZERO_VERSION_COLUMN_NAME,\n updateReplicationWatermark,\n} from './schema/replication-state.ts';\nimport {TableMetadataTracker} from './schema/table-metadata.ts';\n\nexport type ChangeProcessorMode = ReplicatorMode \| 'initial-sync';\n\nexport type CommitResult = {\n watermark: string;\n completedBackfill: DownloadStatus \| undefined;\n schemaUpdated: boolean;\n changeLogUpdated: boolean;\n};\n\n/*\n The ChangeProcessor partitions the stream of messages into transactions\n * by creating a {@link TransactionProcessor} when a transaction begins, and dispatching\n * messages to it until the commit is received.\n \n From https://www.postgresql.org/docs/current/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW :\n \n \"The logical replication protocol sends individual transactions one by one.\n * This means that all messages between a pair of Begin and Commit messages\n * belong to the same transaction.\"\n /\nexport class ChangeProcessor {\n readonly #db: StatementRunner;\n readonly #changeLog: ChangeLog;\n readonly #tableMetadata: TableMetadataTracker;\n readonly #mode: ChangeProcessorMode;\n readonly #failService: (lc: LogContext, err: unknown) => void;\n\n // The TransactionProcessor lazily loads table specs into this Map,\n // and reloads them after a schema change. It is cached here to avoid\n // reading them from the DB on every transaction.\n readonly #tableSpecs = new Map<string, LiteTableSpecWithReplicationStatus>();\n\n #currentTx: TransactionProcessor \| null = null;\n\n #failure: Error \| undefined;\n\n constructor(\n db: StatementRunner,\n mode: ChangeProcessorMode,\n failService: (lc: LogContext, err: unknown) => void,\n ) {\n this.#db = db;\n this.#changeLog = new ChangeLog(db.db);\n this.#tableMetadata = new TableMetadataTracker(db.db);\n this.#mode = mode;\n this.#failService = failService;\n }\n\n #fail(lc: LogContext, err: unknown) {\n if (!this.#failure) {\n this.#currentTx?.abort(lc); // roll back any pending transaction.\n\n this.#failure = ensureError(err);\n\n if (!(err instanceof AbortError)) {\n // Propagate the failure up to the service.\n lc.error?.('Message Processing failed:', this.#failure);\n this.#failService(lc, this.#failure);\n }\n }\n }\n\n abort(lc: LogContext) {\n this.#fail(lc, new AbortError());\n }\n\n /* @return If a transaction was committed. /\n processMessage(\n lc: LogContext,\n downstream: ChangeStreamData,\n ): CommitResult \| null {\n const [type, message] = downstream;\n if (this.#failure) {\n lc.debug?.(`Dropping ${message.tag}`);\n return null;\n }\n try {\n const watermark =\n type === 'begin'\n ? downstream[2].commitWatermark\n : type === 'commit'\n ? downstream[2].watermark\n : undefined;\n return this.#processMessage(lc, message, watermark);\n } catch (e) {\n this.#fail(lc, e);\n }\n return null;\n }\n\n #beginTransaction(\n lc: LogContext,\n commitVersion: string,\n jsonFormat: JSONFormat,\n ): TransactionProcessor {\n const start = Date.now();\n\n // litestream can technically hold the lock for an arbitrary amount of time\n // when checkpointing a large commit. Crashing on the busy-timeout in this\n // scenario will either produce a corrupt backup or otherwise prevent\n // replication from proceeding.\n //\n // Instead, retry the lock acquisition indefinitely. If this masks\n // an unknown deadlock situation, manual intervention will be necessary.\n for (let i = 0; ; i++) {\n try {\n return new TransactionProcessor(\n lc,\n this.#db,\n this.#mode,\n this.#changeLog,\n this.#tableMetadata,\n this.#tableSpecs,\n commitVersion,\n jsonFormat,\n );\n } catch (e) {\n if (e instanceof SqliteError && e.code === 'SQLITE_BUSY') {\n lc.warn?.(\n `SQLITE_BUSY for ${Date.now() - start} ms (attempt ${i + 1}). ` +\n `This is only expected if litestream is performing a large ` +\n `checkpoint.`,\n e,\n );\n continue;\n }\n throw e;\n }\n }\n }\n\n /* @return If a transaction was committed. /\n #processMessage(\n lc: LogContext,\n msg: Change,\n watermark: string \| undefined,\n ): CommitResult \| null {\n if (msg.tag === 'begin') {\n if (this.#currentTx) {\n throw new Error(`Already in a transaction ${stringify(msg)}`);\n }\n this.#currentTx = this.#beginTransaction(\n lc,\n must(watermark),\n msg.json ?? JSON_PARSED,\n );\n return null;\n }\n\n // For non-begin messages, there should be a #currentTx set.\n const tx = this.#currentTx;\n if (!tx) {\n throw new Error(\n `Received message outside of transaction: ${stringify(msg)}`,\n );\n }\n\n if (msg.tag === 'commit') {\n // Undef this.#currentTx to allow the assembly of the next transaction.\n this.#currentTx = null;\n\n assert(watermark, 'watermark is required for commit messages');\n return tx.processCommit(msg, watermark);\n }\n\n if (msg.tag === 'rollback') {\n this.#currentTx?.abort(lc);\n this.#currentTx = null;\n return null;\n }\n\n switch (msg.tag) {\n case 'insert':\n tx.processInsert(msg);\n break;\n case 'update':\n tx.processUpdate(msg);\n break;\n case 'delete':\n tx.processDelete(msg);\n break;\n case 'truncate':\n tx.processTruncate(msg);\n break;\n case 'create-table':\n tx.processCreateTable(msg);\n break;\n case 'rename-table':\n tx.processRenameTable(msg);\n break;\n case 'update-table-metadata':\n tx.processTableMetadata(msg);\n break;\n case 'add-column':\n tx.processAddColumn(msg);\n break;\n case 'update-column':\n tx.processUpdateColumn(msg);\n break;\n case 'drop-column':\n tx.processDropColumn(msg);\n break;\n case 'drop-table':\n tx.processDropTable(msg);\n break;\n case 'create-index':\n tx.processCreateIndex(msg);\n break;\n case 'drop-index':\n tx.processDropIndex(msg);\n break;\n case 'backfill':\n tx.processBackfill(msg);\n break;\n case 'backfill-completed':\n tx.processBackfillCompleted(msg);\n break;\n default:\n unreachable(msg);\n }\n\n return null;\n }\n}\n\n/\n The {@link TransactionProcessor} handles the sequence of messages from\n * upstream, from `BEGIN` to `COMMIT` and executes the corresponding mutations\n * on the {@link postgres.TransactionSql} on the replica.\n \n When applying row contents to the replica, the `_0_version` column is added / updated,\n * and a corresponding entry in the `ChangeLog` is added. The version value is derived\n * from the watermark of the preceding transaction (stored as the `nextStateVersion` in the\n * `ReplicationState` table).\n \n Side note: For non-streaming Postgres transactions, the commitEndLsn (and thus\n * commit watermark) is available in the `begin` message, so it could theoretically\n * be used for the row version of changes within the transaction. However, the\n * commitEndLsn is not available in the streaming (in-progress) transaction\n * protocol, and may not be available for CDC streams of other upstream types.\n * Therefore, the zero replication protocol is designed to not require the commit\n * watermark when a transaction begins.\n \n Also of interest is the fact that all INSERT Messages are logically applied as\n * UPSERTs. See {@link processInsert} for the underlying motivation.\n /\nclass TransactionProcessor {\n readonly #lc: LogContext;\n readonly #startMs: number;\n readonly #db: StatementRunner;\n readonly #mode: ChangeProcessorMode;\n readonly #version: LexiVersion;\n readonly #changeLog: ChangeLog;\n readonly #tableMetadata: TableMetadataTracker;\n readonly #tableSpecs: Map<string, LiteTableSpecWithReplicationStatus>;\n readonly #jsonFormat: JSONFormat;\n readonly #columnMetadata: ColumnMetadataStore;\n\n #pos = 0;\n #schemaChanged = false;\n #numChangeLogEntries = 0;\n\n constructor(\n lc: LogContext,\n db: StatementRunner,\n mode: ChangeProcessorMode,\n changeLog: ChangeLog,\n tableMetadata: TableMetadataTracker,\n tableSpecs: Map<string, LiteTableSpecWithReplicationStatus>,\n commitVersion: LexiVersion,\n jsonFormat: JSONFormat,\n ) {\n this.#startMs = Date.now();\n this.#mode = mode;\n this.#jsonFormat = jsonFormat;\n\n switch (mode) {\n case 'serving':\n // Although the Replicator / Incremental Syncer is the only writer of the replica,\n // a `BEGIN CONCURRENT` transaction is used to allow View Syncers to simulate\n // (i.e. and `ROLLBACK`) changes on historic snapshots of the database for the\n // purpose of IVM).\n //\n // This TransactionProcessor is the only logic that will actually\n // `COMMIT` any transactions to the replica.\n db.beginConcurrent();\n break;\n case 'backup':\n // For the backup-replicator (i.e. replication-manager), there are no View Syncers\n // and thus BEGIN CONCURRENT is not necessary. In fact, BEGIN CONCURRENT can cause\n // deadlocks with forced wal-checkpoints (which `litestream replicate` performs),\n // so it is important to use vanilla transactions in this configuration.\n db.beginImmediate();\n break;\n case 'initial-sync':\n // When the ChangeProcessor is used for initial-sync, the calling code\n // handles the transaction boundaries.\n break;\n default:\n unreachable();\n }\n this.#db = db;\n this.#version = commitVersion;\n this.#lc = lc.withContext('version', commitVersion);\n this.#changeLog = changeLog;\n this.#tableMetadata = tableMetadata;\n this.#tableSpecs = tableSpecs;\n // The column_metadata table is guaranteed to exist since the\n // replica-schema.ts migration to v8.\n this.#columnMetadata = must(ColumnMetadataStore.getInstance(db.db));\n\n if (this.#tableSpecs.size === 0) {\n this.#reloadTableSpecs();\n }\n }\n\n #reloadTableSpecs() {\n this.#tableSpecs.clear();\n // zqlSpecs include the primary key derived from unique indexes\n const zqlSpecs = computeZqlSpecs(this.#lc, this.#db.db, {\n includeBackfillingColumns: true,\n });\n for (let spec of listTables(this.#db.db)) {\n if (!spec.primaryKey) {\n spec = {\n ...spec,\n primaryKey: [\n ...(zqlSpecs.get(spec.name)?.tableSpec.primaryKey ?? []),\n ],\n };\n }\n this.#tableSpecs.set(spec.name, spec);\n }\n }\n\n #tableSpec(name: string) {\n return must(this.#tableSpecs.get(name), `Unknown table ${name}`);\n }\n\n #getKey(\n {row, numCols}: {row: LiteRow; numCols: number},\n {relation}: {relation: MessageRelation},\n ): LiteRowKey {\n const keyColumns =\n relation.rowKey.type !== 'full'\n ? relation.rowKey.columns // already a suitable key\n : this.#tableSpec(liteTableName(relation)).primaryKey;\n if (!keyColumns?.length) {\n throw new Error(\n `Cannot replicate table \"${relation.name}\" without a PRIMARY KEY or UNIQUE INDEX`,\n );\n }\n // For the common case (replica identity default), the row is already the\n // key for deletes and updates, in which case a new object can be avoided.\n if (numCols === keyColumns.length) {\n return row;\n }\n const key: Record<string, LiteValueType> = {};\n for (const col of keyColumns) {\n key[col] = row[col];\n }\n return key;\n }\n\n processInsert(insert: MessageInsert) {\n const table = liteTableName(insert.relation);\n const tableSpec = this.#tableSpec(table);\n const newRow = liteRow(insert.new, tableSpec, this.#jsonFormat);\n\n this.#upsert(table, {\n ...newRow.row,\n [ZERO_VERSION_COLUMN_NAME]: this.#version,\n });\n\n if (insert.relation.rowKey.columns.length === 0) {\n // INSERTs can be replicated for rows without a PRIMARY KEY or a\n // UNIQUE INDEX. These are written to the replica but not recorded\n // in the changeLog, because these rows cannot participate in IVM.\n //\n // (Once the table schema has been corrected to include a key, the\n // associated schema change will reset pipelines and data can be\n // loaded via hydration.)\n return;\n }\n const key = this.#getKey(newRow, insert);\n this.#logSetOp(table, key, getBackfilledColumns(newRow.row, tableSpec));\n }\n\n #upsert(table: string, row: LiteRow) {\n const columns = Object.keys(row).map(c => id(c));\n this.#db.run(\n `\n INSERT OR REPLACE INTO ${id(table)} (${columns.join(',')})\n VALUES (${Array.from({length: columns.length}).fill('?').join(',')})\n `,\n Object.values(row),\n );\n }\n\n // Updates by default are applied as UPDATE commands to support partial\n // row specifications from the change source. In particular, this is needed\n // to handle updates for which unchanged TOASTed values are not sent:\n //\n // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html#PROTOCOL-LOGICALREP-MESSAGE-FORMATS-TUPLEDATA\n //\n // However, in certain cases an UPDATE may be received for a row that\n // was not initially synced, such as when, an existing table is added\n // to the app's publication.\n //\n // In order to facilitate \"resumptive\" replication, the logic falls back to\n // an INSERT if the update did not change any rows.\n processUpdate(update: MessageUpdate) {\n const table = liteTableName(update.relation);\n const tableSpec = this.#tableSpec(table);\n const newRow = liteRow(update.new, tableSpec, this.#jsonFormat);\n const row = {...newRow.row, [ZERO_VERSION_COLUMN_NAME]: this.#version};\n\n // update.key is set with the old values if the key has changed.\n const oldKey = update.key\n ? this.#getKey(\n liteRow(update.key, this.#tableSpec(table), this.#jsonFormat),\n update,\n )\n : null;\n const newKey = this.#getKey(newRow, update);\n\n if (oldKey) {\n this.#logDeleteOp(table, oldKey, tableSpec.backfilling);\n }\n this.#logSetOp(table, newKey, getBackfilledColumns(newRow.row, tableSpec));\n\n const currKey = oldKey ?? newKey;\n const conds = Object.keys(currKey).map(col => `${id(col)}=?`);\n const setExprs = Object.keys(row).map(col => `${id(col)}=?`);\n\n const {changes} = this.#db.run(\n `\n UPDATE ${id(table)}\n SET ${setExprs.join(',')}\n WHERE ${conds.join(' AND ')}\n `,\n [...Object.values(row), ...Object.values(currKey)],\n );\n\n // If the UPDATE did not affect any rows, perform an UPSERT of the\n // new row for resumptive replication.\n if (changes === 0) {\n this.#upsert(table, row);\n }\n }\n\n processDelete(del: MessageDelete) {\n const table = liteTableName(del.relation);\n const tableSpec = this.#tableSpec(table);\n const rowKey = this.#getKey(\n liteRow(del.key, tableSpec, this.#jsonFormat),\n del,\n );\n\n this.#delete(table, rowKey);\n this.#logDeleteOp(table, rowKey, tableSpec.backfilling);\n }\n\n #delete(table: string, rowKey: LiteRowKey) {\n const conds = Object.keys(rowKey).map(col => `${id(col)}=?`);\n this.#db.run(\n `DELETE FROM ${id(table)} WHERE ${conds.join(' AND ')}`,\n Object.values(rowKey),\n );\n }\n\n processTruncate(truncate: MessageTruncate) {\n for (const relation of truncate.relations) {\n const table = liteTableName(relation);\n // Update replica data.\n this.#db.run(`DELETE FROM ${id(table)}`);\n\n // Update change log.\n this.#logTruncateOp(table);\n }\n }\n\n processCreateTable(create: TableCreate) {\n if (create.metadata) {\n this.#tableMetadata.setUpstreamMetadata(create.spec, create.metadata);\n }\n const table = mapPostgresToLite(create.spec);\n this.#db.db.exec(createLiteTableStatement(table));\n\n // Write to metadata table\n for (const [colName, colSpec] of Object.entries(create.spec.columns)) {\n this.#columnMetadata.insert(\n table.name,\n colName,\n colSpec,\n create.backfill?.[colName],\n );\n }\n\n if (\n Object.keys(create.backfill ?? {}).length ===\n Object.keys(create.spec.columns).length\n ) {\n this.#reloadTableSpecs();\n } else {\n // Make the table visible immediately unless all of the columns are\n // being backfilled. In the backfill case, the version bump will happen\n // with the backfill is complete.\n this.#logResetOp(table.name);\n }\n this.#lc.info?.(create.tag, table.name);\n }\n\n processTableMetadata(msg: TableUpdateMetadata) {\n this.#tableMetadata.setUpstreamMetadata(msg.table, msg.new);\n }\n\n processRenameTable(rename: TableRename) {\n this.#tableMetadata.rename(rename.old, rename.new);\n\n const oldName = liteTableName(rename.old);\n const newName = liteTableName(rename.new);\n this.#db.db.exec(`ALTER TABLE ${id(oldName)} RENAME TO ${id(newName)}`);\n\n // Rename in metadata table\n this.#columnMetadata.renameTable(oldName, newName);\n\n this.#bumpVersions(rename.new);\n this.#logResetOp(oldName);\n this.#lc.info?.(rename.tag, oldName, newName);\n }\n\n processAddColumn(msg: ColumnAdd) {\n if (msg.tableMetadata) {\n this.#tableMetadata.setUpstreamMetadata(msg.table, msg.tableMetadata);\n }\n const table = liteTableName(msg.table);\n const {name} = msg.column;\n const spec = mapPostgresToLiteColumn(table, msg.column);\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} ADD ${id(name)} ${liteColumnDef(spec)}`,\n );\n\n // Write to metadata table\n this.#columnMetadata.insert(table, name, msg.column.spec, msg.backfill);\n\n if (msg.backfill) {\n this.#reloadTableSpecs();\n } else {\n // Make the new column visible immediately if it's not being backfilled.\n // Otherwise, the version bump will happen with the backfill is complete.\n this.#bumpVersions(msg.table);\n }\n this.#lc.info?.(msg.tag, table, msg.column);\n }\n\n processUpdateColumn(msg: ColumnUpdate) {\n const table = liteTableName(msg.table);\n let oldName = msg.old.name;\n const newName = msg.new.name;\n\n // update-column can ignore defaults because it does not change the values\n // in existing rows.\n //\n // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DESC-SET-DROP-DEFAULT\n //\n // \"The new default value will only apply in subsequent INSERT or UPDATE\n // commands; it does not cause rows already in the table to change.\"\n //\n // This allows support for _changing_ column defaults to any expression,\n // since it does not affect what the replica needs to do.\n const oldSpec = mapPostgresToLiteColumn(table, msg.old, 'ignore-default');\n const newSpec = mapPostgresToLiteColumn(table, msg.new, 'ignore-default');\n\n // The only updates that are relevant are the column name and the data type.\n if (oldName === newName && oldSpec.dataType === newSpec.dataType) {\n this.#lc.info?.(msg.tag, 'no thing to update', oldSpec, newSpec);\n return;\n }\n // If the data type changes, we have to make a new column with the new data type\n // and copy the values over.\n if (oldSpec.dataType !== newSpec.dataType) {\n // Remember (and drop) the indexes that reference the column.\n const indexes = listIndexes(this.#db.db).filter(\n idx => idx.tableName === table && oldName in idx.columns,\n );\n const stmts = indexes.map(idx => `DROP INDEX IF EXISTS ${id(idx.name)};`);\n const tmpName = `tmp.${newName}`;\n stmts.push(`\n ALTER TABLE ${id(table)} ADD ${id(tmpName)} ${liteColumnDef(newSpec)};\n UPDATE ${id(table)} SET ${id(tmpName)} = ${id(oldName)};\n ALTER TABLE ${id(table)} DROP ${id(oldName)};\n `);\n for (const idx of indexes) {\n // Re-create the indexes to reference the new column.\n idx.columns[tmpName] = idx.columns[oldName];\n delete idx.columns[oldName];\n stmts.push(createLiteIndexStatement(idx));\n }\n this.#db.db.exec(stmts.join(''));\n oldName = tmpName;\n }\n if (oldName !== newName) {\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} RENAME ${id(oldName)} TO ${id(newName)}`,\n );\n }\n\n // Update metadata table\n this.#columnMetadata.update(\n table,\n msg.old.name,\n msg.new.name,\n msg.new.spec,\n );\n\n this.#bumpVersions(msg.table);\n this.#lc.info?.(msg.tag, table, msg.new);\n }\n\n processDropColumn(msg: ColumnDrop) {\n const table = liteTableName(msg.table);\n const {column} = msg;\n this.#db.db.exec(`ALTER TABLE ${id(table)} DROP ${id(column)}`);\n\n // Delete from metadata table\n this.#columnMetadata.deleteColumn(table, column);\n\n this.#bumpVersions(msg.table);\n this.#lc.info?.(msg.tag, table, column);\n }\n\n processDropTable(drop: TableDrop) {\n this.#tableMetadata.drop(drop.id);\n\n const name = liteTableName(drop.id);\n this.#db.db.exec(`DROP TABLE IF EXISTS ${id(name)}`);\n\n // Delete from metadata table\n this.#columnMetadata.deleteTable(name);\n\n this.#logResetOp(name);\n this.#lc.info?.(drop.tag, name);\n }\n\n processCreateIndex(create: IndexCreate) {\n const index = mapPostgresToLiteIndex(create.spec);\n this.#db.db.exec(createLiteIndexStatement(index));\n\n // indexes affect tables visibility (e.g. sync-ability is gated on\n // having a unique index), so reset pipelines to refresh table schemas.\n // However, the reset is not necessary if the index is for a table\n // that is not yet visible due to backfilling.\n const tableSpec = must(this.#tableSpecs.get(index.tableName));\n if (\n (tableSpec.backfilling ?? []).length ===\n Object.entries(tableSpec.columns).length - 1 // don't count _0_version\n ) {\n this.#reloadTableSpecs();\n } else {\n this.#logResetOp(index.tableName);\n }\n this.#lc.info?.(create.tag, index.name);\n }\n\n processDropIndex(drop: IndexDrop) {\n const name = liteTableName(drop.id);\n this.#db.db.exec(`DROP INDEX IF EXISTS ${id(name)}`);\n this.#lc.info?.(drop.tag, name);\n }\n\n #bumpVersions(table: Identifier) {\n this.#tableMetadata.setMinRowVersion(table, this.#version);\n this.#logResetOp(liteTableName(table));\n }\n\n /\n @param backfilledColumns `backfilling` columns for which values were set\n /\n #logSetOp(\n table: string,\n key: LiteRowKey,\n backfilledColumns: string[] \| undefined,\n ) {\n // The \"serving\" replicator always writes to the change-log (for IVM).\n // The \"backup\" replicator only needs to write to the change log\n // when writing columns that are being backfilled.\n if (this.#mode === 'serving' \|\| backfilledColumns !== undefined) {\n this.#changeLog.logSetOp(\n this.#version,\n this.#pos++,\n table,\n key,\n backfilledColumns,\n );\n this.#numChangeLogEntries++;\n }\n }\n\n #logDeleteOp(table: string, key: LiteRowKey, backfilling?: string[]) {\n // The \"serving\" replicator always writes to the change-log (for IVM).\n // The \"backup\" replicator only needs to write to the change log\n // when writing columns that are being backfilled.\n if (this.#mode === 'serving' \|\| backfilling?.length) {\n this.#changeLog.logDeleteOp(this.#version, this.#pos++, table, key);\n this.#numChangeLogEntries++;\n }\n }\n\n #logTruncateOp(table: string) {\n if (this.#mode === 'serving') {\n this.#changeLog.logTruncateOp(this.#version, table);\n this.#numChangeLogEntries++;\n }\n }\n\n #logResetOp(table: string) {\n this.#schemaChanged = true;\n if (this.#mode === 'serving') {\n this.#changeLog.logResetOp(this.#version, table);\n this.#numChangeLogEntries++;\n }\n this.#reloadTableSpecs();\n }\n\n processBackfill({relation, watermark, columns, rowValues}: MessageBackfill) {\n const tableName = liteTableName(relation);\n const tableSpec = must(this.#tableSpecs.get(tableName));\n const rowKeyCols = relation.rowKey.columns;\n const cols = [...rowKeyCols, ...columns];\n\n // Common parts of the INSERT sql statement.\n const insertColsStr = [...cols, ZERO_VERSION_COLUMN_NAME].map(id).join(',');\n const qMarks = Array.from({length: cols.length + 1})\n .fill('?')\n .join(',');\n const rowKeyColsStr = rowKeyCols.map(id).join(',');\n\n let backfilled = 0;\n let skipped = 0;\n for (const v of rowValues) {\n const row = liteRow(\n Object.fromEntries(cols.map((c, i) => [c, v[i]])),\n tableSpec,\n this.#jsonFormat,\n );\n const rowKey = this.#getKey(row, {relation});\n const rowOp = this.#changeLog.getLatestRowOp(tableName, rowKey);\n if (rowOp?.op === DEL_OP && rowOp.stateVersion > watermark) {\n skipped++;\n continue; // the row was deleted after the backfill snapshot\n }\n const updates =\n rowOp?.op === SET_OP\n ? cols.filter(\n c => (rowOp.backfillingColumnVersions[c] ?? '') <= watermark,\n )\n : cols;\n if (updates.length === 0) {\n // row already has newer values for all backfilling columns.\n skipped++;\n continue;\n }\n const updateStmts = updates.map(col => `${id(col)}=excluded.${id(col)}`);\n this.#db.run(\n /sql*/ `\n INSERT INTO ${id(tableName)} (${insertColsStr}) VALUES (${qMarks})\n ON CONFLICT (${rowKeyColsStr})\n DO UPDATE SET ${updateStmts.join(',')};\n `,\n ...Object.values(row.row),\n watermark, // the _0_version for new rows (i.e. table backfill)\n );\n backfilled++;\n }\n\n this.#lc.debug?.(\n `backfilled ${backfilled} rows (skipped ${skipped}) into ${tableName}`,\n );\n }\n\n #completedBackfill: DownloadStatus \| undefined;\n\n processBackfillCompleted({relation, columns, status}: BackfillCompleted) {\n const tableName = liteTableName(relation);\n const rowKeyCols = relation.rowKey.columns;\n const cols = [...rowKeyCols, ...columns];\n\n const columnMetadata = must(ColumnMetadataStore.getInstance(this.#db.db));\n for (const col of cols) {\n columnMetadata.clearBackfilling(tableName, col);\n }\n // Given that new columns are being exposed for every row in the table, bump the\n // row version for all rows.\n this.#bumpVersions(relation);\n if (status) {\n this.#completedBackfill = {table: tableName, columns: cols, ...status};\n }\n this.#lc.info?.(`finished backfilling ${tableName}`);\n\n // Note that there is no need to clear the backfillingColumnVersions values\n // in the changeLog. It could theoretically be done for clarity but:\n // (1) it could be non-trivial in terms of latency introduced and\n // (2) the data must be preserved if _other_ columns are in the process\n // of being backfilled\n //\n // Thus, for speed and simplicity, the values are left as is. (Note that\n // subsequent replicated changes to those rows will clear the values if\n // no backfills are in progress).\n }\n\n processCommit(commit: MessageCommit, watermark: string): CommitResult {\n if (watermark !== this.#version) {\n throw new Error(\n `'commit' version ${watermark} does not match 'begin' version ${\n this.#version\n }: ${stringify(commit)}`,\n );\n }\n updateReplicationWatermark(this.#db, watermark);\n\n if (this.#schemaChanged) {\n const start = Date.now();\n this.#db.db.pragma('optimize');\n this.#lc.info?.(\n `PRAGMA optimized after schema change (${Date.now() - start} ms)`,\n );\n }\n\n if (this.#mode !== 'initial-sync') {\n this.#db.commit();\n }\n\n const elapsedMs = Date.now() - this.#startMs;\n this.#lc.debug?.(`Committed tx@${this.#version} (${elapsedMs} ms)`);\n\n return {\n watermark,\n completedBackfill: this.#completedBackfill,\n schemaUpdated: this.#schemaChanged,\n changeLogUpdated: this.#numChangeLogEntries > 0,\n };\n }\n\n abort(lc: LogContext) {\n lc.info?.(`aborting transaction ${this.#version}`);\n this.#db.rollback();\n }\n}\n\nfunction getBackfilledColumns(\n row: LiteRow,\n {backfilling}: LiteTableSpecWithReplicationStatus,\n): string[] \| undefined {\n if (!backfilling?.length) {\n return undefined; // common case\n }\n return backfilling.filter(col => col in row);\n}\n\nfunction ensureError(err: unknown): Error {\n if (err instanceof Error) {\n return err;\n }\n const error = new Error();\n error.cause = err;\n return error;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsFA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CACA;CAKA,8BAAuB,IAAI,KAAiD;CAE5E,aAA0C;CAE1C;CAEA,YACE,IACA,MACA,aACA;AACA,QAAA,KAAW;AACX,QAAA,YAAkB,IAAI,UAAU,GAAG,GAAG;AACtC,QAAA,gBAAsB,IAAI,qBAAqB,GAAG,GAAG;AACrD,QAAA,OAAa;AACb,QAAA,cAAoB;;CAGtB,MAAM,IAAgB,KAAc;AAClC,MAAI,CAAC,MAAA,SAAe;AAClB,SAAA,WAAiB,MAAM,GAAG;AAE1B,SAAA,UAAgB,YAAY,IAAI;AAEhC,OAAI,EAAE,eAAe,aAAa;AAEhC,OAAG,QAAQ,8BAA8B,MAAA,QAAc;AACvD,UAAA,YAAkB,IAAI,MAAA,QAAc;;;;CAK1C,MAAM,IAAgB;AACpB,QAAA,KAAW,IAAI,IAAI,YAAY,CAAC;;;CAIlC,eACE,IACA,YACqB;EACrB,MAAM,CAAC,MAAM,WAAW;AACxB,MAAI,MAAA,SAAe;AACjB,MAAG,QAAQ,YAAY,QAAQ,MAAM;AACrC,UAAO;;AAET,MAAI;GACF,MAAM,YACJ,SAAS,UACL,WAAW,GAAG,kBACd,SAAS,WACP,WAAW,GAAG,YACd,KAAA;AACR,UAAO,MAAA,eAAqB,IAAI,SAAS,UAAU;WAC5C,GAAG;AACV,SAAA,KAAW,IAAI,EAAE;;AAEnB,SAAO;;CAGT,kBACE,IACA,eACA,YACsB;EACtB,MAAM,QAAQ,KAAK,KAAK;AASxB,OAAK,IAAI,IAAI,IAAK,IAChB,KAAI;AACF,UAAO,IAAI,qBACT,IACA,MAAA,IACA,MAAA,MACA,MAAA,WACA,MAAA,eACA,MAAA,YACA,eACA,WACD;WACM,GAAG;AACV,OAAI,aAAa,eAAe,EAAE,SAAS,eAAe;AACxD,OAAG,OACD,mBAAmB,KAAK,KAAK,GAAG,MAAM,eAAe,IAAI,EAAE,2EAG3D,EACD;AACD;;AAEF,SAAM;;;;CAMZ,gBACE,IACA,KACA,WACqB;AACrB,MAAI,IAAI,QAAQ,SAAS;AACvB,OAAI,MAAA,UACF,OAAM,IAAI,MAAM,4BAA4B,UAAU,IAAI,GAAG;AAE/D,SAAA,YAAkB,MAAA,iBAChB,IACA,KAAK,UAAU,EACf,IAAI,QAAA,IACL;AACD,UAAO;;EAIT,MAAM,KAAK,MAAA;AACX,MAAI,CAAC,GACH,OAAM,IAAI,MACR,4CAA4C,UAAU,IAAI,GAC3D;AAGH,MAAI,IAAI,QAAQ,UAAU;AAExB,SAAA,YAAkB;AAElB,UAAO,WAAW,4CAA4C;AAC9D,UAAO,GAAG,cAAc,KAAK,UAAU;;AAGzC,MAAI,IAAI,QAAQ,YAAY;AAC1B,SAAA,WAAiB,MAAM,GAAG;AAC1B,SAAA,YAAkB;AAClB,UAAO;;AAGT,UAAQ,IAAI,KAAZ;GACE,KAAK;AACH,OAAG,cAAc,IAAI;AACrB;GACF,KAAK;AACH,OAAG,cAAc,IAAI;AACrB;GACF,KAAK;AACH,OAAG,cAAc,IAAI;AACrB;GACF,KAAK;AACH,OAAG,gBAAgB,IAAI;AACvB;GACF,KAAK;AACH,OAAG,mBAAmB,IAAI;AAC1B;GACF,KAAK;AACH,OAAG,mBAAmB,IAAI;AAC1B;GACF,KAAK;AACH,OAAG,qBAAqB,IAAI;AAC5B;GACF,KAAK;AACH,OAAG,iBAAiB,IAAI;AACxB;GACF,KAAK;AACH,OAAG,oBAAoB,IAAI;AAC3B;GACF,KAAK;AACH,OAAG,kBAAkB,IAAI;AACzB;GACF,KAAK;AACH,OAAG,iBAAiB,IAAI;AACxB;GACF,KAAK;AACH,OAAG,mBAAmB,IAAI;AAC1B;GACF,KAAK;AACH,OAAG,iBAAiB,IAAI;AACxB;GACF,KAAK;AACH,OAAG,gBAAgB,IAAI;AACvB;GACF,KAAK;AACH,OAAG,yBAAyB,IAAI;AAChC;GACF,QACE,aAAY,IAAI;;AAGpB,SAAO;;;;;;;;;;;;;;;;;;;;;;;;AAyBX,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,OAAO;CACP,iBAAiB;CACjB,uBAAuB;CAEvB,YACE,IACA,IACA,MACA,WACA,eACA,YACA,eACA,YACA;AACA,QAAA,UAAgB,KAAK,KAAK;AAC1B,QAAA,OAAa;AACb,QAAA,aAAmB;AAEnB,UAAQ,MAAR;GACE,KAAK;AAQH,OAAG,iBAAiB;AACpB;GACF,KAAK;AAKH,OAAG,gBAAgB;AACnB;GACF,KAAK,eAGH;GACF,QACE,cAAa;;AAEjB,QAAA,KAAW;AACX,QAAA,UAAgB;AAChB,QAAA,KAAW,GAAG,YAAY,WAAW,cAAc;AACnD,QAAA,YAAkB;AAClB,QAAA,gBAAsB;AACtB,QAAA,aAAmB;AAGnB,QAAA,iBAAuB,KAAK,oBAAoB,YAAY,GAAG,GAAG,CAAC;AAEnE,MAAI,MAAA,WAAiB,SAAS,EAC5B,OAAA,kBAAwB;;CAI5B,oBAAoB;AAClB,QAAA,WAAiB,OAAO;EAExB,MAAM,WAAW,gBAAgB,MAAA,IAAU,MAAA,GAAS,IAAI,EACtD,2BAA2B,MAC5B,CAAC;AACF,OAAK,IAAI,QAAQ,WAAW,MAAA,GAAS,GAAG,EAAE;AACxC,OAAI,CAAC,KAAK,WACR,QAAO;IACL,GAAG;IACH,YAAY,CACV,GAAI,SAAS,IAAI,KAAK,KAAK,EAAE,UAAU,cAAc,EAAE,CACxD;IACF;AAEH,SAAA,WAAiB,IAAI,KAAK,MAAM,KAAK;;;CAIzC,WAAW,MAAc;AACvB,SAAO,KAAK,MAAA,WAAiB,IAAI,KAAK,EAAE,iBAAiB,OAAO;;CAGlE,QACE,EAAC,KAAK,WACN,EAAC,YACW;EACZ,MAAM,aACJ,SAAS,OAAO,SAAS,SACrB,SAAS,OAAO,UAChB,MAAA,UAAgB,cAAc,SAAS,CAAC,CAAC;AAC/C,MAAI,CAAC,YAAY,OACf,OAAM,IAAI,MACR,2BAA2B,SAAS,KAAK,yCAC1C;AAIH,MAAI,YAAY,WAAW,OACzB,QAAO;EAET,MAAM,MAAqC,EAAE;AAC7C,OAAK,MAAM,OAAO,WAChB,KAAI,OAAO,IAAI;AAEjB,SAAO;;CAGT,cAAc,QAAuB;EACnC,MAAM,QAAQ,cAAc,OAAO,SAAS;EAC5C,MAAM,YAAY,MAAA,UAAgB,MAAM;EACxC,MAAM,SAAS,QAAQ,OAAO,KAAK,WAAW,MAAA,WAAiB;AAE/D,QAAA,OAAa,OAAO;GAClB,GAAG,OAAO;IACT,2BAA2B,MAAA;GAC7B,CAAC;AAEF,MAAI,OAAO,SAAS,OAAO,QAAQ,WAAW,EAQ5C;EAEF,MAAM,MAAM,MAAA,OAAa,QAAQ,OAAO;AACxC,QAAA,SAAe,OAAO,KAAK,qBAAqB,OAAO,KAAK,UAAU,CAAC;;CAGzE,QAAQ,OAAe,KAAc;EACnC,MAAM,UAAU,OAAO,KAAK,IAAI,CAAC,KAAI,MAAK,GAAG,EAAE,CAAC;AAChD,QAAA,GAAS,IACP;+BACyB,GAAG,MAAM,CAAC,IAAI,QAAQ,KAAK,IAAI,CAAC;kBAC7C,MAAM,KAAK,EAAC,QAAQ,QAAQ,QAAO,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC;SAErE,OAAO,OAAO,IAAI,CACnB;;CAeH,cAAc,QAAuB;EACnC,MAAM,QAAQ,cAAc,OAAO,SAAS;EAC5C,MAAM,YAAY,MAAA,UAAgB,MAAM;EACxC,MAAM,SAAS,QAAQ,OAAO,KAAK,WAAW,MAAA,WAAiB;EAC/D,MAAM,MAAM;GAAC,GAAG,OAAO;IAAM,2BAA2B,MAAA;GAAc;EAGtE,MAAM,SAAS,OAAO,MAClB,MAAA,OACE,QAAQ,OAAO,KAAK,MAAA,UAAgB,MAAM,EAAE,MAAA,WAAiB,EAC7D,OACD,GACD;EACJ,MAAM,SAAS,MAAA,OAAa,QAAQ,OAAO;AAE3C,MAAI,OACF,OAAA,YAAkB,OAAO,QAAQ,UAAU,YAAY;AAEzD,QAAA,SAAe,OAAO,QAAQ,qBAAqB,OAAO,KAAK,UAAU,CAAC;EAE1E,MAAM,UAAU,UAAU;EAC1B,MAAM,QAAQ,OAAO,KAAK,QAAQ,CAAC,KAAI,QAAO,GAAG,GAAG,IAAI,CAAC,IAAI;EAC7D,MAAM,WAAW,OAAO,KAAK,IAAI,CAAC,KAAI,QAAO,GAAG,GAAG,IAAI,CAAC,IAAI;EAE5D,MAAM,EAAC,YAAW,MAAA,GAAS,IACzB;eACS,GAAG,MAAM,CAAC;cACX,SAAS,KAAK,IAAI,CAAC;gBACjB,MAAM,KAAK,QAAQ,CAAC;SAE9B,CAAC,GAAG,OAAO,OAAO,IAAI,EAAE,GAAG,OAAO,OAAO,QAAQ,CAAC,CACnD;AAID,MAAI,YAAY,EACd,OAAA,OAAa,OAAO,IAAI;;CAI5B,cAAc,KAAoB;EAChC,MAAM,QAAQ,cAAc,IAAI,SAAS;EACzC,MAAM,YAAY,MAAA,UAAgB,MAAM;EACxC,MAAM,SAAS,MAAA,OACb,QAAQ,IAAI,KAAK,WAAW,MAAA,WAAiB,EAC7C,IACD;AAED,QAAA,OAAa,OAAO,OAAO;AAC3B,QAAA,YAAkB,OAAO,QAAQ,UAAU,YAAY;;CAGzD,QAAQ,OAAe,QAAoB;EACzC,MAAM,QAAQ,OAAO,KAAK,OAAO,CAAC,KAAI,QAAO,GAAG,GAAG,IAAI,CAAC,IAAI;AAC5D,QAAA,GAAS,IACP,eAAe,GAAG,MAAM,CAAC,SAAS,MAAM,KAAK,QAAQ,IACrD,OAAO,OAAO,OAAO,CACtB;;CAGH,gBAAgB,UAA2B;AACzC,OAAK,MAAM,YAAY,SAAS,WAAW;GACzC,MAAM,QAAQ,cAAc,SAAS;AAErC,SAAA,GAAS,IAAI,eAAe,GAAG,MAAM,GAAG;AAGxC,SAAA,cAAoB,MAAM;;;CAI9B,mBAAmB,QAAqB;AACtC,MAAI,OAAO,SACT,OAAA,cAAoB,oBAAoB,OAAO,MAAM,OAAO,SAAS;EAEvE,MAAM,QAAQ,kBAAkB,OAAO,KAAK;AAC5C,QAAA,GAAS,GAAG,KAAK,yBAAyB,MAAM,CAAC;AAGjD,OAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,OAAO,KAAK,QAAQ,CAClE,OAAA,eAAqB,OACnB,MAAM,MACN,SACA,SACA,OAAO,WAAW,SACnB;AAGH,MACE,OAAO,KAAK,OAAO,YAAY,EAAE,CAAC,CAAC,WACnC,OAAO,KAAK,OAAO,KAAK,QAAQ,CAAC,OAEjC,OAAA,kBAAwB;MAKxB,OAAA,WAAiB,MAAM,KAAK;AAE9B,QAAA,GAAS,OAAO,OAAO,KAAK,MAAM,KAAK;;CAGzC,qBAAqB,KAA0B;AAC7C,QAAA,cAAoB,oBAAoB,IAAI,OAAO,IAAI,IAAI;;CAG7D,mBAAmB,QAAqB;AACtC,QAAA,cAAoB,OAAO,OAAO,KAAK,OAAO,IAAI;EAElD,MAAM,UAAU,cAAc,OAAO,IAAI;EACzC,MAAM,UAAU,cAAc,OAAO,IAAI;AACzC,QAAA,GAAS,GAAG,KAAK,eAAe,GAAG,QAAQ,CAAC,aAAa,GAAG,QAAQ,GAAG;AAGvE,QAAA,eAAqB,YAAY,SAAS,QAAQ;AAElD,QAAA,aAAmB,OAAO,IAAI;AAC9B,QAAA,WAAiB,QAAQ;AACzB,QAAA,GAAS,OAAO,OAAO,KAAK,SAAS,QAAQ;;CAG/C,iBAAiB,KAAgB;AAC/B,MAAI,IAAI,cACN,OAAA,cAAoB,oBAAoB,IAAI,OAAO,IAAI,cAAc;EAEvE,MAAM,QAAQ,cAAc,IAAI,MAAM;EACtC,MAAM,EAAC,SAAQ,IAAI;EACnB,MAAM,OAAO,wBAAwB,OAAO,IAAI,OAAO;AACvD,QAAA,GAAS,GAAG,KACV,eAAe,GAAG,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,cAAc,KAAK,GAChE;AAGD,QAAA,eAAqB,OAAO,OAAO,MAAM,IAAI,OAAO,MAAM,IAAI,SAAS;AAEvE,MAAI,IAAI,SACN,OAAA,kBAAwB;MAIxB,OAAA,aAAmB,IAAI,MAAM;AAE/B,QAAA,GAAS,OAAO,IAAI,KAAK,OAAO,IAAI,OAAO;;CAG7C,oBAAoB,KAAmB;EACrC,MAAM,QAAQ,cAAc,IAAI,MAAM;EACtC,IAAI,UAAU,IAAI,IAAI;EACtB,MAAM,UAAU,IAAI,IAAI;EAYxB,MAAM,UAAU,wBAAwB,OAAO,IAAI,KAAK,iBAAiB;EACzE,MAAM,UAAU,wBAAwB,OAAO,IAAI,KAAK,iBAAiB;AAGzE,MAAI,YAAY,WAAW,QAAQ,aAAa,QAAQ,UAAU;AAChE,SAAA,GAAS,OAAO,IAAI,KAAK,sBAAsB,SAAS,QAAQ;AAChE;;AAIF,MAAI,QAAQ,aAAa,QAAQ,UAAU;GAEzC,MAAM,UAAU,YAAY,MAAA,GAAS,GAAG,CAAC,QACvC,QAAO,IAAI,cAAc,SAAS,WAAW,IAAI,QAClD;GACD,MAAM,QAAQ,QAAQ,KAAI,QAAO,wBAAwB,GAAG,IAAI,KAAK,CAAC,GAAG;GACzE,MAAM,UAAU,OAAO;AACvB,SAAM,KAAK;sBACK,GAAG,MAAM,CAAC,OAAO,GAAG,QAAQ,CAAC,GAAG,cAAc,QAAQ,CAAC;iBAC5D,GAAG,MAAM,CAAC,OAAO,GAAG,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC;sBACzC,GAAG,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;UAC1C;AACJ,QAAK,MAAM,OAAO,SAAS;AAEzB,QAAI,QAAQ,WAAW,IAAI,QAAQ;AACnC,WAAO,IAAI,QAAQ;AACnB,UAAM,KAAK,yBAAyB,IAAI,CAAC;;AAE3C,SAAA,GAAS,GAAG,KAAK,MAAM,KAAK,GAAG,CAAC;AAChC,aAAU;;AAEZ,MAAI,YAAY,QACd,OAAA,GAAS,GAAG,KACV,eAAe,GAAG,MAAM,CAAC,UAAU,GAAG,QAAQ,CAAC,MAAM,GAAG,QAAQ,GACjE;AAIH,QAAA,eAAqB,OACnB,OACA,IAAI,IAAI,MACR,IAAI,IAAI,MACR,IAAI,IAAI,KACT;AAED,QAAA,aAAmB,IAAI,MAAM;AAC7B,QAAA,GAAS,OAAO,IAAI,KAAK,OAAO,IAAI,IAAI;;CAG1C,kBAAkB,KAAiB;EACjC,MAAM,QAAQ,cAAc,IAAI,MAAM;EACtC,MAAM,EAAC,WAAU;AACjB,QAAA,GAAS,GAAG,KAAK,eAAe,GAAG,MAAM,CAAC,QAAQ,GAAG,OAAO,GAAG;AAG/D,QAAA,eAAqB,aAAa,OAAO,OAAO;AAEhD,QAAA,aAAmB,IAAI,MAAM;AAC7B,QAAA,GAAS,OAAO,IAAI,KAAK,OAAO,OAAO;;CAGzC,iBAAiB,MAAiB;AAChC,QAAA,cAAoB,KAAK,KAAK,GAAG;EAEjC,MAAM,OAAO,cAAc,KAAK,GAAG;AACnC,QAAA,GAAS,GAAG,KAAK,wBAAwB,GAAG,KAAK,GAAG;AAGpD,QAAA,eAAqB,YAAY,KAAK;AAEtC,QAAA,WAAiB,KAAK;AACtB,QAAA,GAAS,OAAO,KAAK,KAAK,KAAK;;CAGjC,mBAAmB,QAAqB;EACtC,MAAM,QAAQ,uBAAuB,OAAO,KAAK;AACjD,QAAA,GAAS,GAAG,KAAK,yBAAyB,MAAM,CAAC;EAMjD,MAAM,YAAY,KAAK,MAAA,WAAiB,IAAI,MAAM,UAAU,CAAC;AAC7D,OACG,UAAU,eAAe,EAAE,EAAE,WAC9B,OAAO,QAAQ,UAAU,QAAQ,CAAC,SAAS,EAE3C,OAAA,kBAAwB;MAExB,OAAA,WAAiB,MAAM,UAAU;AAEnC,QAAA,GAAS,OAAO,OAAO,KAAK,MAAM,KAAK;;CAGzC,iBAAiB,MAAiB;EAChC,MAAM,OAAO,cAAc,KAAK,GAAG;AACnC,QAAA,GAAS,GAAG,KAAK,wBAAwB,GAAG,KAAK,GAAG;AACpD,QAAA,GAAS,OAAO,KAAK,KAAK,KAAK;;CAGjC,cAAc,OAAmB;AAC/B,QAAA,cAAoB,iBAAiB,OAAO,MAAA,QAAc;AAC1D,QAAA,WAAiB,cAAc,MAAM,CAAC;;;;;CAMxC,UACE,OACA,KACA,mBACA;AAIA,MAAI,MAAA,SAAe,aAAa,sBAAsB,KAAA,GAAW;AAC/D,SAAA,UAAgB,SACd,MAAA,SACA,MAAA,OACA,OACA,KACA,kBACD;AACD,SAAA;;;CAIJ,aAAa,OAAe,KAAiB,aAAwB;AAInE,MAAI,MAAA,SAAe,aAAa,aAAa,QAAQ;AACnD,SAAA,UAAgB,YAAY,MAAA,SAAe,MAAA,OAAa,OAAO,IAAI;AACnE,SAAA;;;CAIJ,eAAe,OAAe;AAC5B,MAAI,MAAA,SAAe,WAAW;AAC5B,SAAA,UAAgB,cAAc,MAAA,SAAe,MAAM;AACnD,SAAA;;;CAIJ,YAAY,OAAe;AACzB,QAAA,gBAAsB;AACtB,MAAI,MAAA,SAAe,WAAW;AAC5B,SAAA,UAAgB,WAAW,MAAA,SAAe,MAAM;AAChD,SAAA;;AAEF,QAAA,kBAAwB;;CAG1B,gBAAgB,EAAC,UAAU,WAAW,SAAS,aAA6B;EAC1E,MAAM,YAAY,cAAc,SAAS;EACzC,MAAM,YAAY,KAAK,MAAA,WAAiB,IAAI,UAAU,CAAC;EACvD,MAAM,aAAa,SAAS,OAAO;EACnC,MAAM,OAAO,CAAC,GAAG,YAAY,GAAG,QAAQ;EAGxC,MAAM,gBAAgB,CAAC,GAAG,MAAM,yBAAyB,CAAC,IAAI,GAAG,CAAC,KAAK,IAAI;EAC3E,MAAM,SAAS,MAAM,KAAK,EAAC,QAAQ,KAAK,SAAS,GAAE,CAAC,CACjD,KAAK,IAAI,CACT,KAAK,IAAI;EACZ,MAAM,gBAAgB,WAAW,IAAI,GAAG,CAAC,KAAK,IAAI;EAElD,IAAI,aAAa;EACjB,IAAI,UAAU;AACd,OAAK,MAAM,KAAK,WAAW;GACzB,MAAM,MAAM,QACV,OAAO,YAAY,KAAK,KAAK,GAAG,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,EACjD,WACA,MAAA,WACD;GACD,MAAM,SAAS,MAAA,OAAa,KAAK,EAAC,UAAS,CAAC;GAC5C,MAAM,QAAQ,MAAA,UAAgB,eAAe,WAAW,OAAO;AAC/D,OAAI,OAAO,OAAA,OAAiB,MAAM,eAAe,WAAW;AAC1D;AACA;;GAEF,MAAM,UACJ,OAAO,OAAA,MACH,KAAK,QACH,OAAM,MAAM,0BAA0B,MAAM,OAAO,UACpD,GACD;AACN,OAAI,QAAQ,WAAW,GAAG;AAExB;AACA;;GAEF,MAAM,cAAc,QAAQ,KAAI,QAAO,GAAG,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG;AACxE,SAAA,GAAS,IACC;sBACM,GAAG,UAAU,CAAC,IAAI,cAAc,YAAY,OAAO;yBAChD,cAAc;0BACb,YAAY,KAAK,IAAI,CAAC;SAExC,GAAG,OAAO,OAAO,IAAI,IAAI,EACzB,UACD;AACD;;AAGF,QAAA,GAAS,QACP,cAAc,WAAW,iBAAiB,QAAQ,SAAS,YAC5D;;CAGH;CAEA,yBAAyB,EAAC,UAAU,SAAS,UAA4B;EACvE,MAAM,YAAY,cAAc,SAAS;EAEzC,MAAM,OAAO,CAAC,GADK,SAAS,OAAO,SACN,GAAG,QAAQ;EAExC,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,MAAA,GAAS,GAAG,CAAC;AACzE,OAAK,MAAM,OAAO,KAChB,gBAAe,iBAAiB,WAAW,IAAI;AAIjD,QAAA,aAAmB,SAAS;AAC5B,MAAI,OACF,OAAA,oBAA0B;GAAC,OAAO;GAAW,SAAS;GAAM,GAAG;GAAO;AAExE,QAAA,GAAS,OAAO,wBAAwB,YAAY;;CAatD,cAAc,QAAuB,WAAiC;AACpE,MAAI,cAAc,MAAA,QAChB,OAAM,IAAI,MACR,oBAAoB,UAAU,kCAC5B,MAAA,QACD,IAAI,UAAU,OAAO,GACvB;AAEH,6BAA2B,MAAA,IAAU,UAAU;AAE/C,MAAI,MAAA,eAAqB;GACvB,MAAM,QAAQ,KAAK,KAAK;AACxB,SAAA,GAAS,GAAG,OAAO,WAAW;AAC9B,SAAA,GAAS,OACP,yCAAyC,KAAK,KAAK,GAAG,MAAM,MAC7D;;AAGH,MAAI,MAAA,SAAe,eACjB,OAAA,GAAS,QAAQ;EAGnB,MAAM,YAAY,KAAK,KAAK,GAAG,MAAA;AAC/B,QAAA,GAAS,QAAQ,gBAAgB,MAAA,QAAc,IAAI,UAAU,MAAM;AAEnE,SAAO;GACL;GACA,mBAAmB,MAAA;GACnB,eAAe,MAAA;GACf,kBAAkB,MAAA,sBAA4B;GAC/C;;CAGH,MAAM,IAAgB;AACpB,KAAG,OAAO,wBAAwB,MAAA,UAAgB;AAClD,QAAA,GAAS,UAAU;;;AAIvB,SAAS,qBACP,KACA,EAAC,eACqB;AACtB,KAAI,CAAC,aAAa,OAChB;AAEF,QAAO,YAAY,QAAO,QAAO,OAAO,IAAI;;AAG9C,SAAS,YAAY,KAAqB;AACxC,KAAI,eAAe,MACjB,QAAO;CAET,MAAM,wBAAQ,IAAI,OAAO;AACzB,OAAM,QAAQ;AACd,QAAO"}

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;~~AAEjD~~,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,~~EAEL~~,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;~~AAI~~/C,OAAO,KAAK,EAAC,0BAA0B,EAAC,MAAM,yBAAyB,CAAC;AACxE,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;~~AAEhE~~;;;;;;GAMG;AACH,qBAAa,iBAAiB;;gBAoB1B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,eAAe,EAAE,0BAA0B,GAAG,IAAI;IAa9C,GAAG;~~IA6IT~~,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAIjC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO;CAGnC"}
1	+ {"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,EAGL,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;AAK/C,OAAO,KAAK,EAAC,0BAA0B,EAAC,MAAM,yBAAyB,CAAC;AACxE,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAIhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;gBAoB1B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,eAAe,EAAE,0BAA0B,GAAG,IAAI;IAa9C,GAAG;IAsJT,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAIjC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO;CAGnC"}

package/out/zero-cache/src/services/replicator/incremental-sync.js CHANGED Viewed

@@ -1,6 +1,7 @@
+import { AbortError } from "../../../../shared/src/abort-error.js";
 import { RunningState } from "../running-state.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
-import "../change-streamer/change-streamer.js";
+import { errorTypeToReadableName } from "../change-streamer/change-streamer.js";
 import { Notifier } from "./notifier.js";
 import { ReplicationReportRecorder } from "./reporter/recorder.js";
 //#region ../zero-cache/src/services/replicator/incremental-sync.ts
@@ -74,9 +75,11 @@ var IncrementalSyncer = class {
 							}
 							break;
 						}
-						case "error":
-							this.stop(lc, message[1]);
+						case "error": {
+							const { type, message: msg } = message[1];
+							this.stop(lc, new AbortError(`${errorTypeToReadableName(type)}: ${msg}`));
 							break;
+						}
 						default: {
 							const msg = message[1];
 							if (msg.tag === "backfill" && msg.status) {

package/out/zero-cache/src/services/replicator/incremental-sync.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n PROTOCOL_VERSION,\n type ChangeStreamer,\n type Downstream,\n} from '../change-streamer/change-streamer.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport type {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/*\n The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n readonly #lc: LogContext;\n readonly #taskID: string;\n readonly #id: string;\n readonly #changeStreamer: ChangeStreamer;\n readonly #worker: WriteWorkerClient;\n readonly #mode: ReplicatorMode;\n readonly #statusPublisher: ReplicationStatusPublisher \| null;\n readonly #notifier: Notifier;\n readonly #reporter: ReplicationReportRecorder;\n\n readonly #state = new RunningState('IncrementalSyncer');\n\n readonly #replicationEvents = getOrCreateCounter(\n 'replication',\n 'events',\n 'Number of replication events processed',\n );\n\n constructor(\n lc: LogContext,\n taskID: string,\n id: string,\n changeStreamer: ChangeStreamer,\n worker: WriteWorkerClient,\n mode: ReplicatorMode,\n statusPublisher: ReplicationStatusPublisher \| null,\n ) {\n this.#lc = lc;\n this.#taskID = taskID;\n this.#id = id;\n this.#changeStreamer = changeStreamer;\n this.#worker = worker;\n this.#mode = mode;\n this.#statusPublisher = statusPublisher;\n this.#notifier = new Notifier();\n this.#reporter = new ReplicationReportRecorder(lc);\n }\n\n async run() {\n const lc = this.#lc;\n this.#worker.onError(err => this.#state.stop(lc, err));\n lc.info?.(`Starting IncrementalSyncer`);\n const {watermark: initialWatermark} =\n await this.#worker.getSubscriptionState();\n\n // Notify any waiting subscribers that the replica is ready to be read.\n void this.#notifier.notifySubscribers();\n\n while (this.#state.shouldRun()) {\n const {replicaVersion, watermark} =\n await this.#worker.getSubscriptionState();\n\n let downstream: Source<Downstream> \| undefined;\n let unregister = () => {};\n let err: unknown \| undefined;\n\n try {\n downstream = await this.#changeStreamer.subscribe({\n protocolVersion: PROTOCOL_VERSION,\n taskID: this.#taskID,\n id: this.#id,\n mode: this.#mode,\n watermark,\n replicaVersion,\n initial: watermark === initialWatermark,\n });\n this.#state.resetBackoff();\n unregister = this.#state.cancelOnStop(downstream);\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Replicating from ${watermark}`,\n );\n\n let backfillStatus: DownloadStatus \| undefined;\n\n for await (const message of downstream) {\n this.#replicationEvents.add(1);\n switch (message[0]) {\n case 'status': {\n const {lagReport} = message[1];\n if (lagReport) {\n const report: ReplicationReport = {\n nextSendTimeMs: lagReport.nextSendTimeMs,\n };\n if (lagReport.lastTimings) {\n report.lastTimings = {\n ...lagReport.lastTimings,\n replicateTimeMs: Date.now(),\n };\n }\n this.#reporter.record(report);\n }\n break;\n }\n case 'error':\n // Unrecoverable error. Stop the service.\n this.stop(lc, message[1]);\n break;\n default: {\n const msg = message[1];\n if (msg.tag === 'backfill' && msg.status) {\n const {status} = msg;\n if (!backfillStatus) {\n // Start publishing the status every 3 seconds.\n backfillStatus = status;\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilling ${msg.relation.name} table`,\n 3000,\n () =>\n backfillStatus\n ? {\n downloadStatus: [\n {\n ...backfillStatus,\n table: msg.relation.name,\n columns: [\n ...msg.relation.rowKey.columns,\n ...msg.columns,\n ],\n },\n ],\n }\n : {},\n );\n }\n backfillStatus = status; // Update the current status\n }\n\n const result = await this.#worker.processMessage(\n message as ChangeStreamData,\n );\n\n this.#handleResult(lc, result);\n if (result?.completedBackfill) {\n backfillStatus = undefined;\n }\n break;\n }\n }\n }\n this.#worker.abort();\n } catch (e) {\n err = e;\n this.#worker.abort();\n } finally {\n downstream?.cancel();\n unregister();\n this.#statusPublisher?.stop();\n }\n await this.#state.backoff(lc, err);\n }\n lc.info?.('IncrementalSyncer stopped');\n }\n\n #handleResult(lc: LogContext, result: CommitResult \| null) {\n if (!result) {\n return;\n }\n if (result.completedBackfill) {\n // Publish the final status\n const status = result.completedBackfill;\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilled ${status.table} table`,\n 0,\n () => ({downloadStatus: [status]}),\n );\n } else if (result.schemaUpdated) {\n this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n }\n if (result.watermark && result.changeLogUpdated) {\n void this.#notifier.notifySubscribers({state: 'version-ready'});\n }\n }\n\n subscribe(): Source<ReplicaState> {\n return this.#notifier.subscribe();\n }\n\n stop(lc: LogContext, err?: unknown) {\n this.#state.stop(lc, err);\n }\n}\n"],"mappings":";;;;;;;;;;;;;AA0BA,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,oBAAoB;CAEvD,qBAA8B,mBAC5B,eACA,UACA,yCACD;CAED,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA;AACA,QAAA,KAAW;AACX,QAAA,SAAe;AACf,QAAA,KAAW;AACX,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,OAAa;AACb,QAAA,kBAAwB;AACxB,QAAA,WAAiB,IAAI,UAAU;AAC/B,QAAA,WAAiB,IAAI,0BAA0B,GAAG;;CAGpD,MAAM,MAAM;EACV,MAAM,KAAK,MAAA;AACX,QAAA,OAAa,SAAQ,QAAO,MAAA,MAAY,KAAK,IAAI,IAAI,CAAC;AACtD,KAAG,OAAO,6BAA6B;EACvC,MAAM,EAAC,WAAW,qBAChB,MAAM,MAAA,OAAa,sBAAsB;AAGtC,QAAA,SAAe,mBAAmB;AAEvC,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,MAAA,OAAa,sBAAsB;GAE3C,IAAI;GACJ,IAAI,mBAAmB;GACvB,IAAI;AAEJ,OAAI;AACF,iBAAa,MAAM,MAAA,eAAqB,UAAU;KAChD,iBAAA;KACA,QAAQ,MAAA;KACR,IAAI,MAAA;KACJ,MAAM,MAAA;KACN;KACA;KACA,SAAS,cAAc;KACxB,CAAC;AACF,UAAA,MAAY,cAAc;AAC1B,iBAAa,MAAA,MAAY,aAAa,WAAW;AACjD,UAAA,iBAAuB,QACrB,IACA,eACA,oBAAoB,YACrB;IAED,IAAI;AAEJ,eAAW,MAAM,WAAW,YAAY;AACtC,WAAA,kBAAwB,IAAI,EAAE;AAC9B,aAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;AAC5B,WAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,gBAC3B;AACD,YAAI,UAAU,YACZ,QAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,KAAK;SAC5B;AAEH,cAAA,SAAe,OAAO,OAAO;;AAE/B;;MAEF,KAAK;AAEH,YAAK,KAAK,IAAI,QAAQ,GAAG;AACzB;MACF,SAAS;OACP,MAAM,MAAM,QAAQ;AACpB,WAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,WAAU;AACjB,YAAI,CAAC,gBAAgB;AAEnB,0BAAiB;AACjB,eAAA,iBAAuB,QACrB,IACA,eACA,eAAe,IAAI,SAAS,KAAK,SACjC,WAEE,iBACI,EACE,gBAAgB,CACd;UACE,GAAG;UACH,OAAO,IAAI,SAAS;UACpB,SAAS,CACP,GAAG,IAAI,SAAS,OAAO,SACvB,GAAG,IAAI,QACR;UACF,CACF,EACF,GACD,EAAE,CACT;;AAEH,yBAAiB;;OAGnB,MAAM,SAAS,MAAM,MAAA,OAAa,eAChC,QACD;AAED,aAAA,aAAmB,IAAI,OAAO;AAC9B,WAAI,QAAQ,kBACV,kBAAiB,KAAA;AAEnB;;;;AAIN,UAAA,OAAa,OAAO;YACb,GAAG;AACV,UAAM;AACN,UAAA,OAAa,OAAO;aACZ;AACR,gBAAY,QAAQ;AACpB,gBAAY;AACZ,UAAA,iBAAuB,MAAM;;AAE/B,SAAM,MAAA,MAAY,QAAQ,IAAI,IAAI;;AAEpC,KAAG,OAAO,4BAA4B;;CAGxC,cAAc,IAAgB,QAA6B;AACzD,MAAI,CAAC,OACH;AAEF,MAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;AACtB,SAAA,iBAAuB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,OAAO,EAAC,EAClC;aACQ,OAAO,cAChB,OAAA,iBAAuB,QAAQ,IAAI,eAAe,iBAAiB;AAErE,MAAI,OAAO,aAAa,OAAO,iBACxB,OAAA,SAAe,kBAAkB,EAAC,OAAO,iBAAgB,CAAC;;CAInE,YAAkC;AAChC,SAAO,MAAA,SAAe,WAAW;;CAGnC,KAAK,IAAgB,KAAe;AAClC,QAAA,MAAY,KAAK,IAAI,IAAI"}
1	+ {"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n errorTypeToReadableName,\n PROTOCOL_VERSION,\n type ChangeStreamer,\n type Downstream,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport type {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/*\n The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n readonly #lc: LogContext;\n readonly #taskID: string;\n readonly #id: string;\n readonly #changeStreamer: ChangeStreamer;\n readonly #worker: WriteWorkerClient;\n readonly #mode: ReplicatorMode;\n readonly #statusPublisher: ReplicationStatusPublisher \| null;\n readonly #notifier: Notifier;\n readonly #reporter: ReplicationReportRecorder;\n\n readonly #state = new RunningState('IncrementalSyncer');\n\n readonly #replicationEvents = getOrCreateCounter(\n 'replication',\n 'events',\n 'Number of replication events processed',\n );\n\n constructor(\n lc: LogContext,\n taskID: string,\n id: string,\n changeStreamer: ChangeStreamer,\n worker: WriteWorkerClient,\n mode: ReplicatorMode,\n statusPublisher: ReplicationStatusPublisher \| null,\n ) {\n this.#lc = lc;\n this.#taskID = taskID;\n this.#id = id;\n this.#changeStreamer = changeStreamer;\n this.#worker = worker;\n this.#mode = mode;\n this.#statusPublisher = statusPublisher;\n this.#notifier = new Notifier();\n this.#reporter = new ReplicationReportRecorder(lc);\n }\n\n async run() {\n const lc = this.#lc;\n this.#worker.onError(err => this.#state.stop(lc, err));\n lc.info?.(`Starting IncrementalSyncer`);\n const {watermark: initialWatermark} =\n await this.#worker.getSubscriptionState();\n\n // Notify any waiting subscribers that the replica is ready to be read.\n void this.#notifier.notifySubscribers();\n\n while (this.#state.shouldRun()) {\n const {replicaVersion, watermark} =\n await this.#worker.getSubscriptionState();\n\n let downstream: Source<Downstream> \| undefined;\n let unregister = () => {};\n let err: unknown \| undefined;\n\n try {\n downstream = await this.#changeStreamer.subscribe({\n protocolVersion: PROTOCOL_VERSION,\n taskID: this.#taskID,\n id: this.#id,\n mode: this.#mode,\n watermark,\n replicaVersion,\n initial: watermark === initialWatermark,\n });\n this.#state.resetBackoff();\n unregister = this.#state.cancelOnStop(downstream);\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Replicating from ${watermark}`,\n );\n\n let backfillStatus: DownloadStatus \| undefined;\n\n for await (const message of downstream) {\n this.#replicationEvents.add(1);\n switch (message[0]) {\n case 'status': {\n const {lagReport} = message[1];\n if (lagReport) {\n const report: ReplicationReport = {\n nextSendTimeMs: lagReport.nextSendTimeMs,\n };\n if (lagReport.lastTimings) {\n report.lastTimings = {\n ...lagReport.lastTimings,\n replicateTimeMs: Date.now(),\n };\n }\n this.#reporter.record(report);\n }\n break;\n }\n case 'error': {\n // Signal from the replication-manager that the view-syncer must\n // shut down and restore a new backup from litestream.\n const {type, message: msg} = message[1];\n this.stop(\n lc,\n // Note: The AbortError indicates a clean / intentional shutdown.\n new AbortError(\n `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n ),\n );\n break;\n }\n default: {\n const msg = message[1];\n if (msg.tag === 'backfill' && msg.status) {\n const {status} = msg;\n if (!backfillStatus) {\n // Start publishing the status every 3 seconds.\n backfillStatus = status;\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilling ${msg.relation.name} table`,\n 3000,\n () =>\n backfillStatus\n ? {\n downloadStatus: [\n {\n ...backfillStatus,\n table: msg.relation.name,\n columns: [\n ...msg.relation.rowKey.columns,\n ...msg.columns,\n ],\n },\n ],\n }\n : {},\n );\n }\n backfillStatus = status; // Update the current status\n }\n\n const result = await this.#worker.processMessage(\n message as ChangeStreamData,\n );\n\n this.#handleResult(lc, result);\n if (result?.completedBackfill) {\n backfillStatus = undefined;\n }\n break;\n }\n }\n }\n this.#worker.abort();\n } catch (e) {\n err = e;\n this.#worker.abort();\n } finally {\n downstream?.cancel();\n unregister();\n this.#statusPublisher?.stop();\n }\n await this.#state.backoff(lc, err);\n }\n lc.info?.('IncrementalSyncer stopped');\n }\n\n #handleResult(lc: LogContext, result: CommitResult \| null) {\n if (!result) {\n return;\n }\n if (result.completedBackfill) {\n // Publish the final status\n const status = result.completedBackfill;\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilled ${status.table} table`,\n 0,\n () => ({downloadStatus: [status]}),\n );\n } else if (result.schemaUpdated) {\n this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n }\n if (result.watermark && result.changeLogUpdated) {\n void this.#notifier.notifySubscribers({state: 'version-ready'});\n }\n }\n\n subscribe(): Source<ReplicaState> {\n return this.#notifier.subscribe();\n }\n\n stop(lc: LogContext, err?: unknown) {\n this.#state.stop(lc, err);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;AAgCA,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,oBAAoB;CAEvD,qBAA8B,mBAC5B,eACA,UACA,yCACD;CAED,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA;AACA,QAAA,KAAW;AACX,QAAA,SAAe;AACf,QAAA,KAAW;AACX,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,OAAa;AACb,QAAA,kBAAwB;AACxB,QAAA,WAAiB,IAAI,UAAU;AAC/B,QAAA,WAAiB,IAAI,0BAA0B,GAAG;;CAGpD,MAAM,MAAM;EACV,MAAM,KAAK,MAAA;AACX,QAAA,OAAa,SAAQ,QAAO,MAAA,MAAY,KAAK,IAAI,IAAI,CAAC;AACtD,KAAG,OAAO,6BAA6B;EACvC,MAAM,EAAC,WAAW,qBAChB,MAAM,MAAA,OAAa,sBAAsB;AAGtC,QAAA,SAAe,mBAAmB;AAEvC,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,MAAA,OAAa,sBAAsB;GAE3C,IAAI;GACJ,IAAI,mBAAmB;GACvB,IAAI;AAEJ,OAAI;AACF,iBAAa,MAAM,MAAA,eAAqB,UAAU;KAChD,iBAAA;KACA,QAAQ,MAAA;KACR,IAAI,MAAA;KACJ,MAAM,MAAA;KACN;KACA;KACA,SAAS,cAAc;KACxB,CAAC;AACF,UAAA,MAAY,cAAc;AAC1B,iBAAa,MAAA,MAAY,aAAa,WAAW;AACjD,UAAA,iBAAuB,QACrB,IACA,eACA,oBAAoB,YACrB;IAED,IAAI;AAEJ,eAAW,MAAM,WAAW,YAAY;AACtC,WAAA,kBAAwB,IAAI,EAAE;AAC9B,aAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;AAC5B,WAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,gBAC3B;AACD,YAAI,UAAU,YACZ,QAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,KAAK;SAC5B;AAEH,cAAA,SAAe,OAAO,OAAO;;AAE/B;;MAEF,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;AACrC,YAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,KAAkB,CAAC,IAAI,MACnD,CACF;AACD;;MAEF,SAAS;OACP,MAAM,MAAM,QAAQ;AACpB,WAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,WAAU;AACjB,YAAI,CAAC,gBAAgB;AAEnB,0BAAiB;AACjB,eAAA,iBAAuB,QACrB,IACA,eACA,eAAe,IAAI,SAAS,KAAK,SACjC,WAEE,iBACI,EACE,gBAAgB,CACd;UACE,GAAG;UACH,OAAO,IAAI,SAAS;UACpB,SAAS,CACP,GAAG,IAAI,SAAS,OAAO,SACvB,GAAG,IAAI,QACR;UACF,CACF,EACF,GACD,EAAE,CACT;;AAEH,yBAAiB;;OAGnB,MAAM,SAAS,MAAM,MAAA,OAAa,eAChC,QACD;AAED,aAAA,aAAmB,IAAI,OAAO;AAC9B,WAAI,QAAQ,kBACV,kBAAiB,KAAA;AAEnB;;;;AAIN,UAAA,OAAa,OAAO;YACb,GAAG;AACV,UAAM;AACN,UAAA,OAAa,OAAO;aACZ;AACR,gBAAY,QAAQ;AACpB,gBAAY;AACZ,UAAA,iBAAuB,MAAM;;AAE/B,SAAM,MAAA,MAAY,QAAQ,IAAI,IAAI;;AAEpC,KAAG,OAAO,4BAA4B;;CAGxC,cAAc,IAAgB,QAA6B;AACzD,MAAI,CAAC,OACH;AAEF,MAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;AACtB,SAAA,iBAAuB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,OAAO,EAAC,EAClC;aACQ,OAAO,cAChB,OAAA,iBAAuB,QAAQ,IAAI,eAAe,iBAAiB;AAErE,MAAI,OAAO,aAAa,OAAO,iBACxB,OAAA,SAAe,kBAAkB,EAAC,OAAO,iBAAgB,CAAC;;CAInE,YAAkC;AAChC,SAAO,MAAA,SAAe,WAAW;;CAGnC,KAAK,IAAgB,KAAe;AAClC,QAAA,MAAY,KAAK,IAAI,IAAI"}

package/out/zero-cache/src/services/replicator/replication-status.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"replication-status.js","names":["#dbRunner","#publish","#timer"],"sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {createSilentLogContext} from '../../../../shared/src/logging-test-utils.ts';\nimport type {JSONObject} from '../../../../zero-events/src/json.ts';\nimport type {\n ReplicatedIndex,\n ReplicatedTable,\n ReplicationStage,\n ReplicationState,\n ReplicationStatusEvent,\n Status,\n} from '../../../../zero-events/src/status.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs, listIndexes} from '../../db/lite-tables.ts';\nimport type {LiteTableSpec} from '../../db/specs.ts';\nimport {\n makeErrorDetails,\n publishCriticalEvent,\n} from '../../observability/events.ts';\n\nconst byKeys = (a: [string, unknown], b: [string, unknown]) =>\n a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;\n\nexport class ReplicationStatusPublisher {\n readonly #dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T;\n readonly #publish: typeof publishCriticalEvent;\n #timer: NodeJS.Timeout \| undefined;\n\n static forTesting() {\n return ReplicationStatusPublisher.forReplicaFile(':memory:');\n }\n\n static forRunningTransaction(tx: Database, publishFn = publishCriticalEvent) {\n return new ReplicationStatusPublisher((_lc, fn) => fn(tx), publishFn);\n }\n\n static forReplicaFile(file: string, publishFn = publishCriticalEvent) {\n return new ReplicationStatusPublisher((lc, fn) => {\n const db = new Database(lc, file, {readonly: true});\n try {\n return fn(db);\n } finally {\n db.close();\n }\n }, publishFn);\n }\n\n constructor(\n dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T,\n publishFn = publishCriticalEvent,\n ) {\n this.#dbRunner = dbRunner;\n this.#publish = publishFn;\n }\n\n publish(\n lc: LogContext,\n stage: ReplicationStage,\n description?: string,\n interval = 0,\n extraState?: () => Partial<ReplicationState>,\n now = new Date(),\n ): this {\n this.stop();\n\n const event = this.#dbRunner(lc, db =>\n replicationStatusEvent(lc, db, stage, 'OK', description, now),\n );\n if (event.state) {\n event.state = {\n ...event.state,\n ...extraState?.(),\n };\n }\n void this.#publish(lc, event);\n\n if (interval) {\n this.#timer = setInterval(\n () => this.publish(lc, stage, description, interval, extraState),\n interval,\n );\n }\n return this;\n }\n\n async publishAndThrowError(\n lc: LogContext,\n stage: ReplicationStage,\n e: unknown,\n ): Promise<never> {\n this.stop();\n const event = this.#dbRunner(lc, db =>\n replicationStatusError(lc, stage, e, db),\n );\n await this.#publish(lc, event);\n throw e;\n }\n\n stop(): this {\n clearInterval(this.#timer);\n return this;\n }\n}\n\nexport async function publishReplicationError(\n lc: LogContext,\n stage: ReplicationStage,\n description: string,\n errorDetails?: JSONObject,\n now = new Date(),\n) {\n const event: ReplicationStatusEvent = {\n type: 'zero/events/status/replication/v1',\n component: 'replication',\n status: 'ERROR',\n stage,\n description,\n errorDetails,\n time: now.toISOString(),\n };\n await publishCriticalEvent(lc, event);\n}\n\nexport function replicationStatusError(\n lc: LogContext,\n stage: ReplicationStage,\n e: unknown,\n db?: Database,\n now = new Date(),\n) {\n const event = replicationStatusEvent(lc, db, stage, 'ERROR', String(e), now);\n event.errorDetails = makeErrorDetails(e);\n return event;\n}\n\n// Exported for testing.\nexport function replicationStatusEvent(\n lc: LogContext,\n db: Database \| undefined,\n stage: ReplicationStage,\n status: Status,\n description?: string,\n now = new Date(),\n): ReplicationStatusEvent {\n const start = performance.now();\n try {\n return {\n type: 'zero/events/status/replication/v1',\n component: 'replication',\n status,\n stage,\n description,\n time: now.toISOString(),\n state: {\n tables: db ? getReplicatedTables(db) : [],\n indexes: db ? getReplicatedIndexes(db) : [],\n replicaSize: db ? getReplicaSize(db) : undefined,\n },\n };\n } catch (e) {\n lc.warn?.(`Unable to create full ReplicationStatusEvent`, e);\n return {\n type: 'zero/events/status/replication/v1',\n component: 'replication',\n status,\n stage,\n description,\n time: now.toISOString(),\n state: {\n tables: [],\n indexes: [],\n replicaSize: 0,\n },\n };\n } finally {\n const elapsed = (performance.now() - start).toFixed(3);\n lc.debug?.(`computed schema for replication event (${elapsed} ms)`);\n }\n}\n\nfunction getReplicatedTables(db: Database): ReplicatedTable[] {\n const fullTables = new Map<string, LiteTableSpec>();\n const clientSchema = computeZqlSpecs(\n createSilentLogContext(), // avoid logging warnings about indexes\n db,\n {includeBackfillingColumns: false},\n new Map(),\n fullTables,\n );\n\n return [...fullTables.entries()].sort(byKeys).map(([table, spec]) => ({\n table,\n columns: Object.entries(spec.columns)\n .sort(byKeys)\n .map(([column, spec]) => ({\n column,\n upstreamType: spec.dataType.split('\|')[0],\n clientType: clientSchema.get(table)?.zqlSpec[column]?.type ?? null,\n })),\n }));\n}\n\nfunction getReplicatedIndexes(db: Database): ReplicatedIndex[] {\n return listIndexes(db).map(({tableName: table, columns, unique}) => ({\n table,\n unique,\n columns: Object.entries(columns)\n .sort(byKeys)\n .map(([column, dir]) => ({column, dir})),\n }));\n}\n\nfunction getReplicaSize(db: Database) {\n const [{page_count: pageCount}] = db.pragma<{page_count: number}>(\n 'page_count',\n );\n const [{page_size: pageSize}] = db.pragma<{page_size: number}>('page_size');\n return pageCount * pageSize;\n}\n"],"mappings":";;;;;AAmBA,IAAM,UAAU,GAAsB,MACpC,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI;AAEvC,IAAa,6BAAb,MAAa,2BAA2B;CACtC;CACA;CACA;CAEA,OAAO,aAAa;AAClB,SAAO,2BAA2B,eAAe,WAAW;;CAG9D,OAAO,sBAAsB,IAAc,YAAY,sBAAsB;AAC3E,SAAO,IAAI,4BAA4B,KAAK,OAAO,GAAG,GAAG,EAAE,UAAU;;CAGvE,OAAO,eAAe,MAAc,YAAY,sBAAsB;AACpE,SAAO,IAAI,4BAA4B,IAAI,OAAO;GAChD,MAAM,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,MAAK,CAAC;AACnD,OAAI;AACF,WAAO,GAAG,GAAG;aACL;AACR,OAAG,OAAO;;KAEX,UAAU;;CAGf,YACE,UACA,YAAY,sBACZ;AACA,QAAA,WAAiB;AACjB,QAAA,UAAgB;;CAGlB,QACE,IACA,OACA,aACA,WAAW,GACX,YACA,sBAAM,IAAI,MAAM,EACV;AACN,OAAK,MAAM;EAEX,MAAM,QAAQ,MAAA,SAAe,KAAI,OAC/B,uBAAuB,IAAI,IAAI,OAAO,MAAM,aAAa,IAAI,CAC9D;AACD,MAAI,MAAM,MACR,OAAM,QAAQ;GACZ,GAAG,MAAM;GACT,GAAG,cAAc;GAClB;AAEE,QAAA,QAAc,IAAI,MAAM;AAE7B,MAAI,SACF,OAAA,QAAc,kBACN,KAAK,QAAQ,IAAI,OAAO,aAAa,UAAU,WAAW,EAChE,SACD;AAEH,SAAO;;CAGT,MAAM,qBACJ,IACA,OACA,GACgB;AAChB,OAAK,MAAM;EACX,MAAM,QAAQ,MAAA,SAAe,KAAI,OAC/B,uBAAuB,IAAI,OAAO,GAAG,GAAG,CACzC;AACD,QAAM,MAAA,QAAc,IAAI,MAAM;AAC9B,QAAM;;CAGR,OAAa;AACX,gBAAc,MAAA,MAAY;AAC1B,SAAO;;;AAIX,eAAsB,wBACpB,IACA,OACA,aACA,cACA,sBAAM,IAAI,MAAM,EAChB;AAUA,OAAM,qBAAqB,IATW;EACpC,MAAM;EACN,WAAW;EACX,QAAQ;EACR;EACA;EACA;EACA,MAAM,IAAI,aAAa;EACxB,CACoC;;AAGvC,SAAgB,uBACd,IACA,OACA,GACA,IACA,sBAAM,IAAI,MAAM,EAChB;CACA,MAAM,QAAQ,uBAAuB,IAAI,IAAI,OAAO,SAAS,OAAO,EAAE,EAAE,IAAI;AAC5E,OAAM,eAAe,iBAAiB,EAAE;AACxC,QAAO;;AAIT,SAAgB,uBACd,IACA,IACA,OACA,QACA,aACA,sBAAM,IAAI,MAAM,EACQ;CACxB,MAAM,QAAQ,YAAY,KAAK;AAC/B,KAAI;AACF,SAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,aAAa;GACvB,OAAO;IACL,QAAQ,KAAK,oBAAoB,GAAG,GAAG,EAAE;IACzC,SAAS,KAAK,qBAAqB,GAAG,GAAG,EAAE;IAC3C,aAAa,KAAK,eAAe,GAAG,GAAG,KAAA;IACxC;GACF;UACM,GAAG;AACV,KAAG,OAAO,gDAAgD,EAAE;AAC5D,SAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,aAAa;GACvB,OAAO;IACL,QAAQ,EAAE;IACV,SAAS,EAAE;IACX,aAAa;IACd;GACF;WACO;EACR,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,KAAG,QAAQ,0CAA0C,QAAQ,MAAM;;;AAIvE,SAAS,oBAAoB,IAAiC;CAC5D,MAAM,6BAAa,IAAI,KAA4B;CACnD,MAAM,eAAe,gBACnB,wBAAwB,EACxB,IACA,EAAC,2BAA2B,OAAM,kBAClC,IAAI,KAAK,EACT,WACD;~~AAED~~,QAAO,CAAC,GAAG,WAAW,SAAS,CAAC,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,OAAO,WAAW;EACpE;EACA,SAAS,OAAO,QAAQ,KAAK,QAAQ,CAClC,KAAK,OAAO,CACZ,KAAK,CAAC,QAAQ,WAAW;GACxB;GACA,cAAc,KAAK,SAAS,MAAM,IAAI,CAAC;GACvC,YAAY,aAAa,IAAI,MAAM,EAAE,QAAQ,SAAS,QAAQ;GAC/D,EAAE;EACN,EAAE;;AAGL,SAAS,qBAAqB,IAAiC;AAC7D,QAAO,YAAY,GAAG,CAAC,KAAK,EAAC,WAAW,OAAO,SAAS,cAAa;EACnE;EACA;EACA,SAAS,OAAO,QAAQ,QAAQ,CAC7B,KAAK,OAAO,CACZ,KAAK,CAAC,QAAQ,UAAU;GAAC;GAAQ;GAAI,EAAE;EAC3C,EAAE;;AAGL,SAAS,eAAe,IAAc;CACpC,MAAM,CAAC,EAAC,YAAY,eAAc,GAAG,OACnC,aACD;CACD,MAAM,CAAC,EAAC,WAAW,cAAa,GAAG,OAA4B,YAAY;AAC3E,QAAO,YAAY"}
1	+ {"version":3,"file":"replication-status.js","names":["#dbRunner","#publish","#timer"],"sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {createSilentLogContext} from '../../../../shared/src/logging-test-utils.ts';\nimport type {JSONObject} from '../../../../zero-events/src/json.ts';\nimport type {\n ReplicatedIndex,\n ReplicatedTable,\n ReplicationStage,\n ReplicationState,\n ReplicationStatusEvent,\n Status,\n} from '../../../../zero-events/src/status.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs, listIndexes} from '../../db/lite-tables.ts';\nimport type {LiteTableSpec} from '../../db/specs.ts';\nimport {\n makeErrorDetails,\n publishCriticalEvent,\n} from '../../observability/events.ts';\n\nconst byKeys = (a: [string, unknown], b: [string, unknown]) =>\n a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;\n\nexport class ReplicationStatusPublisher {\n readonly #dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T;\n readonly #publish: typeof publishCriticalEvent;\n #timer: NodeJS.Timeout \| undefined;\n\n static forTesting() {\n return ReplicationStatusPublisher.forReplicaFile(':memory:');\n }\n\n static forRunningTransaction(tx: Database, publishFn = publishCriticalEvent) {\n return new ReplicationStatusPublisher((_lc, fn) => fn(tx), publishFn);\n }\n\n static forReplicaFile(file: string, publishFn = publishCriticalEvent) {\n return new ReplicationStatusPublisher((lc, fn) => {\n const db = new Database(lc, file, {readonly: true});\n try {\n return fn(db);\n } finally {\n db.close();\n }\n }, publishFn);\n }\n\n constructor(\n dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T,\n publishFn = publishCriticalEvent,\n ) {\n this.#dbRunner = dbRunner;\n this.#publish = publishFn;\n }\n\n publish(\n lc: LogContext,\n stage: ReplicationStage,\n description?: string,\n interval = 0,\n extraState?: () => Partial<ReplicationState>,\n now = new Date(),\n ): this {\n this.stop();\n\n const event = this.#dbRunner(lc, db =>\n replicationStatusEvent(lc, db, stage, 'OK', description, now),\n );\n if (event.state) {\n event.state = {\n ...event.state,\n ...extraState?.(),\n };\n }\n void this.#publish(lc, event);\n\n if (interval) {\n this.#timer = setInterval(\n () => this.publish(lc, stage, description, interval, extraState),\n interval,\n );\n }\n return this;\n }\n\n async publishAndThrowError(\n lc: LogContext,\n stage: ReplicationStage,\n e: unknown,\n ): Promise<never> {\n this.stop();\n const event = this.#dbRunner(lc, db =>\n replicationStatusError(lc, stage, e, db),\n );\n await this.#publish(lc, event);\n throw e;\n }\n\n stop(): this {\n clearInterval(this.#timer);\n return this;\n }\n}\n\nexport async function publishReplicationError(\n lc: LogContext,\n stage: ReplicationStage,\n description: string,\n errorDetails?: JSONObject,\n now = new Date(),\n) {\n const event: ReplicationStatusEvent = {\n type: 'zero/events/status/replication/v1',\n component: 'replication',\n status: 'ERROR',\n stage,\n description,\n errorDetails,\n time: now.toISOString(),\n };\n await publishCriticalEvent(lc, event);\n}\n\nexport function replicationStatusError(\n lc: LogContext,\n stage: ReplicationStage,\n e: unknown,\n db?: Database,\n now = new Date(),\n) {\n const event = replicationStatusEvent(lc, db, stage, 'ERROR', String(e), now);\n event.errorDetails = makeErrorDetails(e);\n return event;\n}\n\n// Exported for testing.\nexport function replicationStatusEvent(\n lc: LogContext,\n db: Database \| undefined,\n stage: ReplicationStage,\n status: Status,\n description?: string,\n now = new Date(),\n): ReplicationStatusEvent {\n const start = performance.now();\n try {\n return {\n type: 'zero/events/status/replication/v1',\n component: 'replication',\n status,\n stage,\n description,\n time: now.toISOString(),\n state: {\n tables: db ? getReplicatedTables(db) : [],\n indexes: db ? getReplicatedIndexes(db) : [],\n replicaSize: db ? getReplicaSize(db) : undefined,\n },\n };\n } catch (e) {\n lc.warn?.(`Unable to create full ReplicationStatusEvent`, e);\n return {\n type: 'zero/events/status/replication/v1',\n component: 'replication',\n status,\n stage,\n description,\n time: now.toISOString(),\n state: {\n tables: [],\n indexes: [],\n replicaSize: 0,\n },\n };\n } finally {\n const elapsed = (performance.now() - start).toFixed(3);\n lc.debug?.(`computed schema for replication event (${elapsed} ms)`);\n }\n}\n\nfunction getReplicatedTables(db: Database): ReplicatedTable[] {\n const fullTables = new Map<string, LiteTableSpec>();\n const clientSchema = computeZqlSpecs(\n createSilentLogContext(), // avoid logging warnings about indexes\n db,\n {includeBackfillingColumns: false},\n new Map(),\n fullTables,\n );\n\n // oxlint-disable-next-line e18e/prefer-array-to-sorted\n return [...fullTables.entries()].sort(byKeys).map(([table, spec]) => ({\n table,\n columns: Object.entries(spec.columns)\n .sort(byKeys)\n .map(([column, spec]) => ({\n column,\n upstreamType: spec.dataType.split('\|')[0],\n clientType: clientSchema.get(table)?.zqlSpec[column]?.type ?? null,\n })),\n }));\n}\n\nfunction getReplicatedIndexes(db: Database): ReplicatedIndex[] {\n return listIndexes(db).map(({tableName: table, columns, unique}) => ({\n table,\n unique,\n columns: Object.entries(columns)\n .sort(byKeys)\n .map(([column, dir]) => ({column, dir})),\n }));\n}\n\nfunction getReplicaSize(db: Database) {\n const [{page_count: pageCount}] = db.pragma<{page_count: number}>(\n 'page_count',\n );\n const [{page_size: pageSize}] = db.pragma<{page_size: number}>('page_size');\n return pageCount * pageSize;\n}\n"],"mappings":";;;;;AAmBA,IAAM,UAAU,GAAsB,MACpC,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI;AAEvC,IAAa,6BAAb,MAAa,2BAA2B;CACtC;CACA;CACA;CAEA,OAAO,aAAa;AAClB,SAAO,2BAA2B,eAAe,WAAW;;CAG9D,OAAO,sBAAsB,IAAc,YAAY,sBAAsB;AAC3E,SAAO,IAAI,4BAA4B,KAAK,OAAO,GAAG,GAAG,EAAE,UAAU;;CAGvE,OAAO,eAAe,MAAc,YAAY,sBAAsB;AACpE,SAAO,IAAI,4BAA4B,IAAI,OAAO;GAChD,MAAM,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,MAAK,CAAC;AACnD,OAAI;AACF,WAAO,GAAG,GAAG;aACL;AACR,OAAG,OAAO;;KAEX,UAAU;;CAGf,YACE,UACA,YAAY,sBACZ;AACA,QAAA,WAAiB;AACjB,QAAA,UAAgB;;CAGlB,QACE,IACA,OACA,aACA,WAAW,GACX,YACA,sBAAM,IAAI,MAAM,EACV;AACN,OAAK,MAAM;EAEX,MAAM,QAAQ,MAAA,SAAe,KAAI,OAC/B,uBAAuB,IAAI,IAAI,OAAO,MAAM,aAAa,IAAI,CAC9D;AACD,MAAI,MAAM,MACR,OAAM,QAAQ;GACZ,GAAG,MAAM;GACT,GAAG,cAAc;GAClB;AAEE,QAAA,QAAc,IAAI,MAAM;AAE7B,MAAI,SACF,OAAA,QAAc,kBACN,KAAK,QAAQ,IAAI,OAAO,aAAa,UAAU,WAAW,EAChE,SACD;AAEH,SAAO;;CAGT,MAAM,qBACJ,IACA,OACA,GACgB;AAChB,OAAK,MAAM;EACX,MAAM,QAAQ,MAAA,SAAe,KAAI,OAC/B,uBAAuB,IAAI,OAAO,GAAG,GAAG,CACzC;AACD,QAAM,MAAA,QAAc,IAAI,MAAM;AAC9B,QAAM;;CAGR,OAAa;AACX,gBAAc,MAAA,MAAY;AAC1B,SAAO;;;AAIX,eAAsB,wBACpB,IACA,OACA,aACA,cACA,sBAAM,IAAI,MAAM,EAChB;AAUA,OAAM,qBAAqB,IATW;EACpC,MAAM;EACN,WAAW;EACX,QAAQ;EACR;EACA;EACA;EACA,MAAM,IAAI,aAAa;EACxB,CACoC;;AAGvC,SAAgB,uBACd,IACA,OACA,GACA,IACA,sBAAM,IAAI,MAAM,EAChB;CACA,MAAM,QAAQ,uBAAuB,IAAI,IAAI,OAAO,SAAS,OAAO,EAAE,EAAE,IAAI;AAC5E,OAAM,eAAe,iBAAiB,EAAE;AACxC,QAAO;;AAIT,SAAgB,uBACd,IACA,IACA,OACA,QACA,aACA,sBAAM,IAAI,MAAM,EACQ;CACxB,MAAM,QAAQ,YAAY,KAAK;AAC/B,KAAI;AACF,SAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,aAAa;GACvB,OAAO;IACL,QAAQ,KAAK,oBAAoB,GAAG,GAAG,EAAE;IACzC,SAAS,KAAK,qBAAqB,GAAG,GAAG,EAAE;IAC3C,aAAa,KAAK,eAAe,GAAG,GAAG,KAAA;IACxC;GACF;UACM,GAAG;AACV,KAAG,OAAO,gDAAgD,EAAE;AAC5D,SAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,aAAa;GACvB,OAAO;IACL,QAAQ,EAAE;IACV,SAAS,EAAE;IACX,aAAa;IACd;GACF;WACO;EACR,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,KAAG,QAAQ,0CAA0C,QAAQ,MAAM;;;AAIvE,SAAS,oBAAoB,IAAiC;CAC5D,MAAM,6BAAa,IAAI,KAA4B;CACnD,MAAM,eAAe,gBACnB,wBAAwB,EACxB,IACA,EAAC,2BAA2B,OAAM,kBAClC,IAAI,KAAK,EACT,WACD;AAGD,QAAO,CAAC,GAAG,WAAW,SAAS,CAAC,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,OAAO,WAAW;EACpE;EACA,SAAS,OAAO,QAAQ,KAAK,QAAQ,CAClC,KAAK,OAAO,CACZ,KAAK,CAAC,QAAQ,WAAW;GACxB;GACA,cAAc,KAAK,SAAS,MAAM,IAAI,CAAC;GACvC,YAAY,aAAa,IAAI,MAAM,EAAE,QAAQ,SAAS,QAAQ;GAC/D,EAAE;EACN,EAAE;;AAGL,SAAS,qBAAqB,IAAiC;AAC7D,QAAO,YAAY,GAAG,CAAC,KAAK,EAAC,WAAW,OAAO,SAAS,cAAa;EACnE;EACA;EACA,SAAS,OAAO,QAAQ,QAAQ,CAC7B,KAAK,OAAO,CACZ,KAAK,CAAC,QAAQ,UAAU;GAAC;GAAQ;GAAI,EAAE;EAC3C,EAAE;;AAGL,SAAS,eAAe,IAAc;CACpC,MAAM,CAAC,EAAC,YAAY,eAAc,GAAG,OACnC,aACD;CACD,MAAM,CAAC,EAAC,WAAW,cAAa,GAAG,OAA4B,YAAY;AAC3E,QAAO,YAAY"}

package/out/zero-cache/src/services/replicator/schema/column-metadata.d.ts CHANGED Viewed

@@ -40,7 +40,7 @@ export declare class ColumnMetadataStore {
      * Returns `undefined` if the metadata table doesn't exist yet.
      */
     static getInstance(db: Database): ColumnMetadataStore | undefined;
-    insert(tableName: string, columnName: string, spec: ColumnSpec, backfill?: BackfillID | undefined): void;
+    insert(tableName: string, columnName: string, spec: ColumnSpec, backfill?: BackfillID): void;
     update(tableName: string, oldColumnName: string, newColumnName: string, spec: ColumnSpec): void;
     clearBackfilling(tableName: string, columnName: string): void;
     deleteColumn(tableName: string, columnName: string): void;

package/out/zero-cache/src/services/replicator/schema/column-metadata.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"column-metadata.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/column-metadata.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,iCAAiC,CAAC;AAEzE,OAAO,KAAK,EAAC,UAAU,EAAE,aAAa,EAAC,MAAM,sBAAsB,CAAC;AAQpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,yCAAyC,CAAC;AAExE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,2EAA2E;IAC3E,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,sDAAsD;IACtD,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,aAAa,EAAE,OAAO,CAAC;CACxB;AAWD,eAAO,MAAM,4BAA4B,8VAYxC,CAAC;AAEF;;;;;;GAMG;AACH,qBAAa,mBAAmB;;IAa9B,OAAO;IA2DP;;;OAGG;IACH,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,GAAG,mBAAmB,GAAG,SAAS;IAoBjE,MAAM,CACJ,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,UAAU,EAChB,QAAQ,CAAC,EAAE,UAAU,~~GAAG~~,~~SAAS,GAChC,~~IAAI;IAuBP,MAAM,CACJ,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,aAAa,EAAE,MAAM,EACrB,IAAI,EAAE,UAAU,GACf,IAAI;IAcP,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAI7D,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAIzD,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAIpC,WAAW,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,IAAI;IAI7D,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS;IAmB5E,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC;IAoBxD,QAAQ,IAAI,OAAO;CAIpB;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CACxC,EAAE,EAAE,QAAQ,EACZ,MAAM,EAAE,aAAa,EAAE,GACtB,IAAI,CA0BN;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,cAAc,EAAE,MAAM,EACtB,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,GACjC,cAAc,CAkBhB;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,CAOzE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,UAAU,GAAG,cAAc,CASvE"}
1	+ {"version":3,"file":"column-metadata.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/column-metadata.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAC,QAAQ,EAAY,MAAM,iCAAiC,CAAC;AAEzE,OAAO,KAAK,EAAC,UAAU,EAAE,aAAa,EAAC,MAAM,sBAAsB,CAAC;AAQpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,yCAAyC,CAAC;AAExE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,2EAA2E;IAC3E,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,sDAAsD;IACtD,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,aAAa,EAAE,OAAO,CAAC;CACxB;AAWD,eAAO,MAAM,4BAA4B,8VAYxC,CAAC;AAEF;;;;;;GAMG;AACH,qBAAa,mBAAmB;;IAa9B,OAAO;IA2DP;;;OAGG;IACH,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,GAAG,mBAAmB,GAAG,SAAS;IAoBjE,MAAM,CACJ,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,UAAU,EAChB,QAAQ,CAAC,EAAE,UAAU,GACpB,IAAI;IAuBP,MAAM,CACJ,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,aAAa,EAAE,MAAM,EACrB,IAAI,EAAE,UAAU,GACf,IAAI;IAcP,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAI7D,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAIzD,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAIpC,WAAW,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,IAAI;IAI7D,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS;IAmB5E,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC;IAoBxD,QAAQ,IAAI,OAAO;CAIpB;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CACxC,EAAE,EAAE,QAAQ,EACZ,MAAM,EAAE,aAAa,EAAE,GACtB,IAAI,CA0BN;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,cAAc,EAAE,MAAM,EACtB,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,GACjC,cAAc,CAkBhB;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,CAOzE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,UAAU,GAAG,cAAc,CASvE"}

package/out/zero-cache/src/services/replicator/schema/column-metadata.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"column-metadata.js","names":["#instances","#insertStmt","#updateStmt","#clearBackfillStmt","#deleteColumnStmt","#deleteTableStmt","#renameTableStmt","#getColumnStmt","#getTableStmt","#hasTableStmt","#insertMetadata"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/column-metadata.ts"],"sourcesContent":["/*\n Column metadata table for storing upstream PostgreSQL schema information.\n \n Previously, upstream type metadata was embedded in SQLite column type strings\n * using pipe-delimited notation (e.g., \"int8\|NOT_NULL\|TEXT_ENUM\"). This caused\n * issues with SQLite type affinity and made schema inspection difficult.\n \n This table stores that metadata separately, allowing SQLite columns to use\n * plain type names while preserving all necessary upstream type information.\n /\n\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {isArrayColumn, isEnumColumn} from '../../../db/pg-to-lite.ts';\nimport type {ColumnSpec, LiteTableSpec} from '../../../db/specs.ts';\nimport {\n isArray as checkIsArray,\n isEnum as checkIsEnum,\n liteTypeString,\n nullableUpstream,\n upstreamDataType,\n} from '../../../types/lite.ts';\nimport type {BackfillID} from '../../change-source/protocol/current.ts';\n\n/\n Structured column metadata, replacing the old pipe-delimited string format.\n /\nexport interface ColumnMetadata {\n /* PostgreSQL type name, e.g., 'int8', 'varchar', 'text[]', 'user_role' /\n upstreamType: string;\n isNotNull: boolean;\n isEnum: boolean;\n isArray: boolean;\n /* Maximum character length for varchar/char types /\n characterMaxLength?: number \| null;\n isBackfilling: boolean;\n}\n\ntype ColumnMetadataRow = {\n upstream_type: string;\n is_not_null: number;\n is_enum: number;\n is_array: number;\n character_max_length: number \| null;\n backfill: string \| null;\n};\n\nexport const CREATE_COLUMN_METADATA_TABLE = `\n CREATE TABLE \"_zero.column_metadata\" (\n table_name TEXT NOT NULL,\n column_name TEXT NOT NULL,\n upstream_type TEXT NOT NULL,\n is_not_null INTEGER NOT NULL,\n is_enum INTEGER NOT NULL,\n is_array INTEGER NOT NULL,\n character_max_length INTEGER,\n backfill TEXT,\n PRIMARY KEY (table_name, column_name)\n );\n`;\n\n/\n Efficient column metadata store that prepares all statements upfront.\n * Use this class to avoid re-preparing statements on every operation.\n \n Access via `ColumnMetadataStore.getInstance(db)`, which returns `undefined`\n * if the metadata table doesn't exist yet.\n /\nexport class ColumnMetadataStore {\n static #instances = new WeakMap<Database, ColumnMetadataStore>();\n\n readonly #insertStmt: Statement;\n readonly #updateStmt: Statement;\n readonly #clearBackfillStmt: Statement;\n readonly #deleteColumnStmt: Statement;\n readonly #deleteTableStmt: Statement;\n readonly #renameTableStmt: Statement;\n readonly #getColumnStmt: Statement;\n readonly #getTableStmt: Statement;\n readonly #hasTableStmt: Statement;\n\n private constructor(db: Database) {\n this.#insertStmt = db.prepare(`\n INSERT INTO \"_zero.column_metadata\"\n (table_name, column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length, backfill)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?)\n `);\n\n this.#updateStmt = db.prepare(`\n UPDATE \"_zero.column_metadata\"\n SET column_name = ?,\n upstream_type = ?,\n is_not_null = ?,\n is_enum = ?,\n is_array = ?,\n character_max_length = ?\n WHERE table_name = ? AND column_name = ?\n `);\n\n this.#clearBackfillStmt = db.prepare(/sql/ `\n UPDATE \"_zero.column_metadata\"\n SET backfill = NULL\n WHERE table_name = ? AND column_name = ?\n `);\n\n this.#deleteColumnStmt = db.prepare(`\n DELETE FROM \"_zero.column_metadata\"\n WHERE table_name = ? AND column_name = ?\n `);\n\n this.#deleteTableStmt = db.prepare(`\n DELETE FROM \"_zero.column_metadata\"\n WHERE table_name = ?\n `);\n\n this.#renameTableStmt = db.prepare(`\n UPDATE \"_zero.column_metadata\"\n SET table_name = ?\n WHERE table_name = ?\n `);\n\n this.#getColumnStmt = db.prepare(`\n SELECT upstream_type, is_not_null, is_enum, is_array, character_max_length, backfill\n FROM \"_zero.column_metadata\"\n WHERE table_name = ? AND column_name = ?\n `);\n\n this.#getTableStmt = db.prepare(`\n SELECT column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length, backfill\n FROM \"_zero.column_metadata\"\n WHERE table_name = ?\n ORDER BY column_name\n `);\n\n this.#hasTableStmt = db.prepare(`\n SELECT 1 FROM sqlite_master\n WHERE type = 'table' AND name = '_zero.column_metadata'\n `);\n }\n\n /\n Gets the singleton instance of ColumnMetadataStore for the given database.\n * Returns `undefined` if the metadata table doesn't exist yet.\n /\n static getInstance(db: Database): ColumnMetadataStore \| undefined {\n // Check if table exists\n const tableExists = db\n .prepare(\n `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n )\n .get();\n\n if (!tableExists) {\n return undefined;\n }\n\n let instance = ColumnMetadataStore.#instances.get(db);\n if (!instance) {\n instance = new ColumnMetadataStore(db);\n ColumnMetadataStore.#instances.set(db, instance);\n }\n return instance;\n }\n\n insert(\n tableName: string,\n columnName: string,\n spec: ColumnSpec,\n backfill?: BackfillID \| undefined,\n ): void {\n const metadata = pgColumnSpecToMetadata(spec);\n this.#insertMetadata(tableName, columnName, metadata, backfill);\n }\n\n #insertMetadata(\n tableName: string,\n columnName: string,\n metadata: Omit<ColumnMetadata, 'isBackfilling'>,\n backfill?: BackfillID \| undefined,\n ): void {\n this.#insertStmt.run(\n tableName,\n columnName,\n metadata.upstreamType,\n metadata.isNotNull ? 1 : 0,\n metadata.isEnum ? 1 : 0,\n metadata.isArray ? 1 : 0,\n metadata.characterMaxLength ?? null,\n backfill ? JSON.stringify(backfill) : null,\n );\n }\n\n update(\n tableName: string,\n oldColumnName: string,\n newColumnName: string,\n spec: ColumnSpec,\n ): void {\n const metadata = pgColumnSpecToMetadata(spec);\n this.#updateStmt.run(\n newColumnName,\n metadata.upstreamType,\n metadata.isNotNull ? 1 : 0,\n metadata.isEnum ? 1 : 0,\n metadata.isArray ? 1 : 0,\n metadata.characterMaxLength ?? null,\n tableName,\n oldColumnName,\n );\n }\n\n clearBackfilling(tableName: string, columnName: string): void {\n this.#clearBackfillStmt.run(tableName, columnName);\n }\n\n deleteColumn(tableName: string, columnName: string): void {\n this.#deleteColumnStmt.run(tableName, columnName);\n }\n\n deleteTable(tableName: string): void {\n this.#deleteTableStmt.run(tableName);\n }\n\n renameTable(oldTableName: string, newTableName: string): void {\n this.#renameTableStmt.run(newTableName, oldTableName);\n }\n\n getColumn(tableName: string, columnName: string): ColumnMetadata \| undefined {\n const row = this.#getColumnStmt.get(tableName, columnName) as\n \| ColumnMetadataRow\n \| undefined;\n\n if (!row) {\n return undefined;\n }\n\n return {\n upstreamType: row.upstream_type,\n isNotNull: row.is_not_null !== 0,\n isEnum: row.is_enum !== 0,\n isArray: row.is_array !== 0,\n characterMaxLength: row.character_max_length,\n isBackfilling: row.backfill !== null,\n };\n }\n\n getTable(tableName: string): Map<string, ColumnMetadata> {\n const rows = this.#getTableStmt.all(tableName) as Array<\n ColumnMetadataRow & {column_name: string}\n >;\n\n const metadata = new Map<string, ColumnMetadata>();\n for (const row of rows) {\n metadata.set(row.column_name, {\n upstreamType: row.upstream_type,\n isNotNull: row.is_not_null !== 0,\n isEnum: row.is_enum !== 0,\n isArray: row.is_array !== 0,\n characterMaxLength: row.character_max_length,\n isBackfilling: row.backfill !== null,\n });\n }\n\n return metadata;\n }\n\n hasTable(): boolean {\n const result = this.#hasTableStmt.get();\n return result !== undefined;\n }\n}\n\n/\n Populates metadata table from existing tables that use pipe notation.\n * This is used during migration v8 to backfill the metadata table.\n /\nexport function populateFromExistingTables(\n db: Database,\n tables: LiteTableSpec[],\n): void {\n // The backfill column is not relevant here, and does not exist on\n // older versions of the replica.\n const legacyInsertStmt = db.prepare(`\n INSERT INTO \"_zero.column_metadata\"\n (table_name, column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length)\n VALUES (?, ?, ?, ?, ?, ?, ?)\n `);\n\n for (const table of tables) {\n for (const [columnName, columnSpec] of Object.entries(table.columns)) {\n const metadata = liteTypeStringToMetadata(\n columnSpec.dataType,\n columnSpec.characterMaximumLength,\n );\n legacyInsertStmt.run(\n table.name,\n columnName,\n metadata.upstreamType,\n metadata.isNotNull ? 1 : 0,\n metadata.isEnum ? 1 : 0,\n metadata.isArray ? 1 : 0,\n metadata.characterMaxLength ?? null,\n );\n }\n }\n}\n\n/\n Converts pipe-delimited LiteTypeString to structured ColumnMetadata.\n * This is a compatibility helper for the migration period.\n /\nexport function liteTypeStringToMetadata(\n liteTypeString: string,\n characterMaxLength?: number \| null,\n): ColumnMetadata {\n const baseType = upstreamDataType(liteTypeString);\n const isArrayType = checkIsArray(liteTypeString);\n\n // Reconstruct the full upstream type including array notation\n // For new-style arrays like 'text[]', upstreamDataType returns 'text[]'\n // For old-style arrays like 'int4\|NOT_NULL[]', upstreamDataType returns 'int4', so we append '[]'\n const fullUpstreamType =\n isArrayType && !baseType.includes('[]') ? `${baseType}[]` : baseType;\n\n return {\n upstreamType: fullUpstreamType,\n isNotNull: !nullableUpstream(liteTypeString),\n isEnum: checkIsEnum(liteTypeString),\n isArray: isArrayType,\n characterMaxLength: characterMaxLength ?? null,\n isBackfilling: false,\n };\n}\n\n/\n Converts structured ColumnMetadata back to pipe-delimited LiteTypeString.\n * This is a compatibility helper for the migration period.\n /\nexport function metadataToLiteTypeString(metadata: ColumnMetadata): string {\n return liteTypeString(\n metadata.upstreamType,\n metadata.isNotNull,\n metadata.isEnum,\n metadata.isArray,\n );\n}\n\n/\n Converts PostgreSQL ColumnSpec to structured ColumnMetadata.\n * Used during replication to populate the metadata table from upstream schema.\n \n Uses the same logic as liteTypeString() and mapPostgresToLiteColumn() via shared helpers.\n */\nexport function pgColumnSpecToMetadata(spec: ColumnSpec): ColumnMetadata {\n return {\n upstreamType: spec.dataType,\n isNotNull: spec.notNull ?? false,\n isEnum: isEnumColumn(spec),\n isArray: isArrayColumn(spec),\n characterMaxLength: spec.characterMaximumLength ?? null,\n isBackfilling: false,\n };\n}\n"],"mappings":";;;AA8CA,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;AAqB5C,IAAa,sBAAb,MAAa,oBAAoB;CAC/B,QAAA,4BAAoB,IAAI,SAAwC;CAEhE;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAoB,IAAc;AAChC,QAAA,aAAmB,GAAG,QAAQ;;;;MAI5B;AAEF,QAAA,aAAmB,GAAG,QAAQ;;;;;;;;;MAS5B;AAEF,QAAA,oBAA0B,GAAG,QAAgB;;;;MAI3C;AAEF,QAAA,mBAAyB,GAAG,QAAQ;;;MAGlC;AAEF,QAAA,kBAAwB,GAAG,QAAQ;;;MAGjC;AAEF,QAAA,kBAAwB,GAAG,QAAQ;;;;MAIjC;AAEF,QAAA,gBAAsB,GAAG,QAAQ;;;;MAI/B;AAEF,QAAA,eAAqB,GAAG,QAAQ;;;;;MAK9B;AAEF,QAAA,eAAqB,GAAG,QAAQ;;;MAG9B;;;;;;CAOJ,OAAO,YAAY,IAA+C;AAQhE,MAAI,CANgB,GACjB,QACC,sFACD,CACA,KAAK,CAGN;EAGF,IAAI,WAAW,qBAAA,UAA+B,IAAI,GAAG;AACrD,MAAI,CAAC,UAAU;AACb,cAAW,IAAI,oBAAoB,GAAG;AACtC,wBAAA,UAA+B,IAAI,IAAI,SAAS;;AAElD,SAAO;;CAGT,OACE,WACA,YACA,MACA,UACM;EACN,MAAM,WAAW,uBAAuB,KAAK;AAC7C,QAAA,eAAqB,WAAW,YAAY,UAAU,SAAS;;CAGjE,gBACE,WACA,YACA,UACA,UACM;AACN,QAAA,WAAiB,IACf,WACA,YACA,SAAS,cACT,SAAS,YAAY,IAAI,GACzB,SAAS,SAAS,IAAI,GACtB,SAAS,UAAU,IAAI,GACvB,SAAS,sBAAsB,MAC/B,WAAW,KAAK,UAAU,SAAS,GAAG,KACvC;;CAGH,OACE,WACA,eACA,eACA,MACM;EACN,MAAM,WAAW,uBAAuB,KAAK;AAC7C,QAAA,WAAiB,IACf,eACA,SAAS,cACT,SAAS,YAAY,IAAI,GACzB,SAAS,SAAS,IAAI,GACtB,SAAS,UAAU,IAAI,GACvB,SAAS,sBAAsB,MAC/B,WACA,cACD;;CAGH,iBAAiB,WAAmB,YAA0B;AAC5D,QAAA,kBAAwB,IAAI,WAAW,WAAW;;CAGpD,aAAa,WAAmB,YAA0B;AACxD,QAAA,iBAAuB,IAAI,WAAW,WAAW;;CAGnD,YAAY,WAAyB;AACnC,QAAA,gBAAsB,IAAI,UAAU;;CAGtC,YAAY,cAAsB,cAA4B;AAC5D,QAAA,gBAAsB,IAAI,cAAc,aAAa;;CAGvD,UAAU,WAAmB,YAAgD;EAC3E,MAAM,MAAM,MAAA,cAAoB,IAAI,WAAW,WAAW;AAI1D,MAAI,CAAC,IACH;AAGF,SAAO;GACL,cAAc,IAAI;GAClB,WAAW,IAAI,gBAAgB;GAC/B,QAAQ,IAAI,YAAY;GACxB,SAAS,IAAI,aAAa;GAC1B,oBAAoB,IAAI;GACxB,eAAe,IAAI,aAAa;GACjC;;CAGH,SAAS,WAAgD;EACvD,MAAM,OAAO,MAAA,aAAmB,IAAI,UAAU;EAI9C,MAAM,2BAAW,IAAI,KAA6B;AAClD,OAAK,MAAM,OAAO,KAChB,UAAS,IAAI,IAAI,aAAa;GAC5B,cAAc,IAAI;GAClB,WAAW,IAAI,gBAAgB;GAC/B,QAAQ,IAAI,YAAY;GACxB,SAAS,IAAI,aAAa;GAC1B,oBAAoB,IAAI;GACxB,eAAe,IAAI,aAAa;GACjC,CAAC;AAGJ,SAAO;;CAGT,WAAoB;AAElB,SADe,MAAA,aAAmB,KAAK,KACrB,KAAA;;;;;;;AAQtB,SAAgB,2BACd,IACA,QACM;CAGN,MAAM,mBAAmB,GAAG,QAAQ;;;;MAIhC;AAEJ,MAAK,MAAM,SAAS,OAClB,MAAK,MAAM,CAAC,YAAY,eAAe,OAAO,QAAQ,MAAM,QAAQ,EAAE;EACpE,MAAM,WAAW,yBACf,WAAW,UACX,WAAW,uBACZ;AACD,mBAAiB,IACf,MAAM,MACN,YACA,SAAS,cACT,SAAS,YAAY,IAAI,GACzB,SAAS,SAAS,IAAI,GACtB,SAAS,UAAU,IAAI,GACvB,SAAS,sBAAsB,KAChC;;;;;;;AASP,SAAgB,yBACd,gBACA,oBACgB;CAChB,MAAM,WAAW,iBAAiB,eAAe;CACjD,MAAM,cAAc,QAAa,eAAe;AAQhD,QAAO;EACL,cAHA,eAAe,CAAC,SAAS,SAAS,KAAK,GAAG,GAAG,SAAS,MAAM;EAI5D,WAAW,CAAC,iBAAiB,eAAe;EAC5C,QAAQ,OAAY,eAAe;EACnC,SAAS;EACT,oBAAoB,sBAAsB;EAC1C,eAAe;EAChB;;;;;;AAOH,SAAgB,yBAAyB,UAAkC;AACzE,QAAO,eACL,SAAS,cACT,SAAS,WACT,SAAS,QACT,SAAS,QACV;;;;;;;;AASH,SAAgB,uBAAuB,MAAkC;AACvE,QAAO;EACL,cAAc,KAAK;EACnB,WAAW,KAAK,WAAW;EAC3B,QAAQ,aAAa,KAAK;EAC1B,SAAS,cAAc,KAAK;EAC5B,oBAAoB,KAAK,0BAA0B;EACnD,eAAe;EAChB"}
1	+ {"version":3,"file":"column-metadata.js","names":["#instances","#insertStmt","#updateStmt","#clearBackfillStmt","#deleteColumnStmt","#deleteTableStmt","#renameTableStmt","#getColumnStmt","#getTableStmt","#hasTableStmt","#insertMetadata"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/column-metadata.ts"],"sourcesContent":["/*\n Column metadata table for storing upstream PostgreSQL schema information.\n \n Previously, upstream type metadata was embedded in SQLite column type strings\n * using pipe-delimited notation (e.g., \"int8\|NOT_NULL\|TEXT_ENUM\"). This caused\n * issues with SQLite type affinity and made schema inspection difficult.\n \n This table stores that metadata separately, allowing SQLite columns to use\n * plain type names while preserving all necessary upstream type information.\n /\n\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {isArrayColumn, isEnumColumn} from '../../../db/pg-to-lite.ts';\nimport type {ColumnSpec, LiteTableSpec} from '../../../db/specs.ts';\nimport {\n isArray as checkIsArray,\n isEnum as checkIsEnum,\n liteTypeString,\n nullableUpstream,\n upstreamDataType,\n} from '../../../types/lite.ts';\nimport type {BackfillID} from '../../change-source/protocol/current.ts';\n\n/\n Structured column metadata, replacing the old pipe-delimited string format.\n /\nexport interface ColumnMetadata {\n /* PostgreSQL type name, e.g., 'int8', 'varchar', 'text[]', 'user_role' /\n upstreamType: string;\n isNotNull: boolean;\n isEnum: boolean;\n isArray: boolean;\n /* Maximum character length for varchar/char types /\n characterMaxLength?: number \| null;\n isBackfilling: boolean;\n}\n\ntype ColumnMetadataRow = {\n upstream_type: string;\n is_not_null: number;\n is_enum: number;\n is_array: number;\n character_max_length: number \| null;\n backfill: string \| null;\n};\n\nexport const CREATE_COLUMN_METADATA_TABLE = `\n CREATE TABLE \"_zero.column_metadata\" (\n table_name TEXT NOT NULL,\n column_name TEXT NOT NULL,\n upstream_type TEXT NOT NULL,\n is_not_null INTEGER NOT NULL,\n is_enum INTEGER NOT NULL,\n is_array INTEGER NOT NULL,\n character_max_length INTEGER,\n backfill TEXT,\n PRIMARY KEY (table_name, column_name)\n );\n`;\n\n/\n Efficient column metadata store that prepares all statements upfront.\n * Use this class to avoid re-preparing statements on every operation.\n \n Access via `ColumnMetadataStore.getInstance(db)`, which returns `undefined`\n * if the metadata table doesn't exist yet.\n /\nexport class ColumnMetadataStore {\n static #instances = new WeakMap<Database, ColumnMetadataStore>();\n\n readonly #insertStmt: Statement;\n readonly #updateStmt: Statement;\n readonly #clearBackfillStmt: Statement;\n readonly #deleteColumnStmt: Statement;\n readonly #deleteTableStmt: Statement;\n readonly #renameTableStmt: Statement;\n readonly #getColumnStmt: Statement;\n readonly #getTableStmt: Statement;\n readonly #hasTableStmt: Statement;\n\n private constructor(db: Database) {\n this.#insertStmt = db.prepare(`\n INSERT INTO \"_zero.column_metadata\"\n (table_name, column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length, backfill)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?)\n `);\n\n this.#updateStmt = db.prepare(`\n UPDATE \"_zero.column_metadata\"\n SET column_name = ?,\n upstream_type = ?,\n is_not_null = ?,\n is_enum = ?,\n is_array = ?,\n character_max_length = ?\n WHERE table_name = ? AND column_name = ?\n `);\n\n this.#clearBackfillStmt = db.prepare(/sql/ `\n UPDATE \"_zero.column_metadata\"\n SET backfill = NULL\n WHERE table_name = ? AND column_name = ?\n `);\n\n this.#deleteColumnStmt = db.prepare(`\n DELETE FROM \"_zero.column_metadata\"\n WHERE table_name = ? AND column_name = ?\n `);\n\n this.#deleteTableStmt = db.prepare(`\n DELETE FROM \"_zero.column_metadata\"\n WHERE table_name = ?\n `);\n\n this.#renameTableStmt = db.prepare(`\n UPDATE \"_zero.column_metadata\"\n SET table_name = ?\n WHERE table_name = ?\n `);\n\n this.#getColumnStmt = db.prepare(`\n SELECT upstream_type, is_not_null, is_enum, is_array, character_max_length, backfill\n FROM \"_zero.column_metadata\"\n WHERE table_name = ? AND column_name = ?\n `);\n\n this.#getTableStmt = db.prepare(`\n SELECT column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length, backfill\n FROM \"_zero.column_metadata\"\n WHERE table_name = ?\n ORDER BY column_name\n `);\n\n this.#hasTableStmt = db.prepare(`\n SELECT 1 FROM sqlite_master\n WHERE type = 'table' AND name = '_zero.column_metadata'\n `);\n }\n\n /\n Gets the singleton instance of ColumnMetadataStore for the given database.\n * Returns `undefined` if the metadata table doesn't exist yet.\n /\n static getInstance(db: Database): ColumnMetadataStore \| undefined {\n // Check if table exists\n const tableExists = db\n .prepare(\n `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n )\n .get();\n\n if (!tableExists) {\n return undefined;\n }\n\n let instance = ColumnMetadataStore.#instances.get(db);\n if (!instance) {\n instance = new ColumnMetadataStore(db);\n ColumnMetadataStore.#instances.set(db, instance);\n }\n return instance;\n }\n\n insert(\n tableName: string,\n columnName: string,\n spec: ColumnSpec,\n backfill?: BackfillID,\n ): void {\n const metadata = pgColumnSpecToMetadata(spec);\n this.#insertMetadata(tableName, columnName, metadata, backfill);\n }\n\n #insertMetadata(\n tableName: string,\n columnName: string,\n metadata: Omit<ColumnMetadata, 'isBackfilling'>,\n backfill?: BackfillID,\n ): void {\n this.#insertStmt.run(\n tableName,\n columnName,\n metadata.upstreamType,\n metadata.isNotNull ? 1 : 0,\n metadata.isEnum ? 1 : 0,\n metadata.isArray ? 1 : 0,\n metadata.characterMaxLength ?? null,\n backfill ? JSON.stringify(backfill) : null,\n );\n }\n\n update(\n tableName: string,\n oldColumnName: string,\n newColumnName: string,\n spec: ColumnSpec,\n ): void {\n const metadata = pgColumnSpecToMetadata(spec);\n this.#updateStmt.run(\n newColumnName,\n metadata.upstreamType,\n metadata.isNotNull ? 1 : 0,\n metadata.isEnum ? 1 : 0,\n metadata.isArray ? 1 : 0,\n metadata.characterMaxLength ?? null,\n tableName,\n oldColumnName,\n );\n }\n\n clearBackfilling(tableName: string, columnName: string): void {\n this.#clearBackfillStmt.run(tableName, columnName);\n }\n\n deleteColumn(tableName: string, columnName: string): void {\n this.#deleteColumnStmt.run(tableName, columnName);\n }\n\n deleteTable(tableName: string): void {\n this.#deleteTableStmt.run(tableName);\n }\n\n renameTable(oldTableName: string, newTableName: string): void {\n this.#renameTableStmt.run(newTableName, oldTableName);\n }\n\n getColumn(tableName: string, columnName: string): ColumnMetadata \| undefined {\n const row = this.#getColumnStmt.get(tableName, columnName) as\n \| ColumnMetadataRow\n \| undefined;\n\n if (!row) {\n return undefined;\n }\n\n return {\n upstreamType: row.upstream_type,\n isNotNull: row.is_not_null !== 0,\n isEnum: row.is_enum !== 0,\n isArray: row.is_array !== 0,\n characterMaxLength: row.character_max_length,\n isBackfilling: row.backfill !== null,\n };\n }\n\n getTable(tableName: string): Map<string, ColumnMetadata> {\n const rows = this.#getTableStmt.all(tableName) as Array<\n ColumnMetadataRow & {column_name: string}\n >;\n\n const metadata = new Map<string, ColumnMetadata>();\n for (const row of rows) {\n metadata.set(row.column_name, {\n upstreamType: row.upstream_type,\n isNotNull: row.is_not_null !== 0,\n isEnum: row.is_enum !== 0,\n isArray: row.is_array !== 0,\n characterMaxLength: row.character_max_length,\n isBackfilling: row.backfill !== null,\n });\n }\n\n return metadata;\n }\n\n hasTable(): boolean {\n const result = this.#hasTableStmt.get();\n return result !== undefined;\n }\n}\n\n/\n Populates metadata table from existing tables that use pipe notation.\n * This is used during migration v8 to backfill the metadata table.\n /\nexport function populateFromExistingTables(\n db: Database,\n tables: LiteTableSpec[],\n): void {\n // The backfill column is not relevant here, and does not exist on\n // older versions of the replica.\n const legacyInsertStmt = db.prepare(`\n INSERT INTO \"_zero.column_metadata\"\n (table_name, column_name, upstream_type, is_not_null, is_enum, is_array, character_max_length)\n VALUES (?, ?, ?, ?, ?, ?, ?)\n `);\n\n for (const table of tables) {\n for (const [columnName, columnSpec] of Object.entries(table.columns)) {\n const metadata = liteTypeStringToMetadata(\n columnSpec.dataType,\n columnSpec.characterMaximumLength,\n );\n legacyInsertStmt.run(\n table.name,\n columnName,\n metadata.upstreamType,\n metadata.isNotNull ? 1 : 0,\n metadata.isEnum ? 1 : 0,\n metadata.isArray ? 1 : 0,\n metadata.characterMaxLength ?? null,\n );\n }\n }\n}\n\n/\n Converts pipe-delimited LiteTypeString to structured ColumnMetadata.\n * This is a compatibility helper for the migration period.\n /\nexport function liteTypeStringToMetadata(\n liteTypeString: string,\n characterMaxLength?: number \| null,\n): ColumnMetadata {\n const baseType = upstreamDataType(liteTypeString);\n const isArrayType = checkIsArray(liteTypeString);\n\n // Reconstruct the full upstream type including array notation\n // For new-style arrays like 'text[]', upstreamDataType returns 'text[]'\n // For old-style arrays like 'int4\|NOT_NULL[]', upstreamDataType returns 'int4', so we append '[]'\n const fullUpstreamType =\n isArrayType && !baseType.includes('[]') ? `${baseType}[]` : baseType;\n\n return {\n upstreamType: fullUpstreamType,\n isNotNull: !nullableUpstream(liteTypeString),\n isEnum: checkIsEnum(liteTypeString),\n isArray: isArrayType,\n characterMaxLength: characterMaxLength ?? null,\n isBackfilling: false,\n };\n}\n\n/\n Converts structured ColumnMetadata back to pipe-delimited LiteTypeString.\n * This is a compatibility helper for the migration period.\n /\nexport function metadataToLiteTypeString(metadata: ColumnMetadata): string {\n return liteTypeString(\n metadata.upstreamType,\n metadata.isNotNull,\n metadata.isEnum,\n metadata.isArray,\n );\n}\n\n/\n Converts PostgreSQL ColumnSpec to structured ColumnMetadata.\n * Used during replication to populate the metadata table from upstream schema.\n \n Uses the same logic as liteTypeString() and mapPostgresToLiteColumn() via shared helpers.\n */\nexport function pgColumnSpecToMetadata(spec: ColumnSpec): ColumnMetadata {\n return {\n upstreamType: spec.dataType,\n isNotNull: spec.notNull ?? false,\n isEnum: isEnumColumn(spec),\n isArray: isArrayColumn(spec),\n characterMaxLength: spec.characterMaximumLength ?? null,\n isBackfilling: false,\n };\n}\n"],"mappings":";;;AA8CA,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;AAqB5C,IAAa,sBAAb,MAAa,oBAAoB;CAC/B,QAAA,4BAAoB,IAAI,SAAwC;CAEhE;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAoB,IAAc;AAChC,QAAA,aAAmB,GAAG,QAAQ;;;;MAI5B;AAEF,QAAA,aAAmB,GAAG,QAAQ;;;;;;;;;MAS5B;AAEF,QAAA,oBAA0B,GAAG,QAAgB;;;;MAI3C;AAEF,QAAA,mBAAyB,GAAG,QAAQ;;;MAGlC;AAEF,QAAA,kBAAwB,GAAG,QAAQ;;;MAGjC;AAEF,QAAA,kBAAwB,GAAG,QAAQ;;;;MAIjC;AAEF,QAAA,gBAAsB,GAAG,QAAQ;;;;MAI/B;AAEF,QAAA,eAAqB,GAAG,QAAQ;;;;;MAK9B;AAEF,QAAA,eAAqB,GAAG,QAAQ;;;MAG9B;;;;;;CAOJ,OAAO,YAAY,IAA+C;AAQhE,MAAI,CANgB,GACjB,QACC,sFACD,CACA,KAAK,CAGN;EAGF,IAAI,WAAW,qBAAA,UAA+B,IAAI,GAAG;AACrD,MAAI,CAAC,UAAU;AACb,cAAW,IAAI,oBAAoB,GAAG;AACtC,wBAAA,UAA+B,IAAI,IAAI,SAAS;;AAElD,SAAO;;CAGT,OACE,WACA,YACA,MACA,UACM;EACN,MAAM,WAAW,uBAAuB,KAAK;AAC7C,QAAA,eAAqB,WAAW,YAAY,UAAU,SAAS;;CAGjE,gBACE,WACA,YACA,UACA,UACM;AACN,QAAA,WAAiB,IACf,WACA,YACA,SAAS,cACT,SAAS,YAAY,IAAI,GACzB,SAAS,SAAS,IAAI,GACtB,SAAS,UAAU,IAAI,GACvB,SAAS,sBAAsB,MAC/B,WAAW,KAAK,UAAU,SAAS,GAAG,KACvC;;CAGH,OACE,WACA,eACA,eACA,MACM;EACN,MAAM,WAAW,uBAAuB,KAAK;AAC7C,QAAA,WAAiB,IACf,eACA,SAAS,cACT,SAAS,YAAY,IAAI,GACzB,SAAS,SAAS,IAAI,GACtB,SAAS,UAAU,IAAI,GACvB,SAAS,sBAAsB,MAC/B,WACA,cACD;;CAGH,iBAAiB,WAAmB,YAA0B;AAC5D,QAAA,kBAAwB,IAAI,WAAW,WAAW;;CAGpD,aAAa,WAAmB,YAA0B;AACxD,QAAA,iBAAuB,IAAI,WAAW,WAAW;;CAGnD,YAAY,WAAyB;AACnC,QAAA,gBAAsB,IAAI,UAAU;;CAGtC,YAAY,cAAsB,cAA4B;AAC5D,QAAA,gBAAsB,IAAI,cAAc,aAAa;;CAGvD,UAAU,WAAmB,YAAgD;EAC3E,MAAM,MAAM,MAAA,cAAoB,IAAI,WAAW,WAAW;AAI1D,MAAI,CAAC,IACH;AAGF,SAAO;GACL,cAAc,IAAI;GAClB,WAAW,IAAI,gBAAgB;GAC/B,QAAQ,IAAI,YAAY;GACxB,SAAS,IAAI,aAAa;GAC1B,oBAAoB,IAAI;GACxB,eAAe,IAAI,aAAa;GACjC;;CAGH,SAAS,WAAgD;EACvD,MAAM,OAAO,MAAA,aAAmB,IAAI,UAAU;EAI9C,MAAM,2BAAW,IAAI,KAA6B;AAClD,OAAK,MAAM,OAAO,KAChB,UAAS,IAAI,IAAI,aAAa;GAC5B,cAAc,IAAI;GAClB,WAAW,IAAI,gBAAgB;GAC/B,QAAQ,IAAI,YAAY;GACxB,SAAS,IAAI,aAAa;GAC1B,oBAAoB,IAAI;GACxB,eAAe,IAAI,aAAa;GACjC,CAAC;AAGJ,SAAO;;CAGT,WAAoB;AAElB,SADe,MAAA,aAAmB,KAAK,KACrB,KAAA;;;;;;;AAQtB,SAAgB,2BACd,IACA,QACM;CAGN,MAAM,mBAAmB,GAAG,QAAQ;;;;MAIhC;AAEJ,MAAK,MAAM,SAAS,OAClB,MAAK,MAAM,CAAC,YAAY,eAAe,OAAO,QAAQ,MAAM,QAAQ,EAAE;EACpE,MAAM,WAAW,yBACf,WAAW,UACX,WAAW,uBACZ;AACD,mBAAiB,IACf,MAAM,MACN,YACA,SAAS,cACT,SAAS,YAAY,IAAI,GACzB,SAAS,SAAS,IAAI,GACtB,SAAS,UAAU,IAAI,GACvB,SAAS,sBAAsB,KAChC;;;;;;;AASP,SAAgB,yBACd,gBACA,oBACgB;CAChB,MAAM,WAAW,iBAAiB,eAAe;CACjD,MAAM,cAAc,QAAa,eAAe;AAQhD,QAAO;EACL,cAHA,eAAe,CAAC,SAAS,SAAS,KAAK,GAAG,GAAG,SAAS,MAAM;EAI5D,WAAW,CAAC,iBAAiB,eAAe;EAC5C,QAAQ,OAAY,eAAe;EACnC,SAAS;EACT,oBAAoB,sBAAsB;EAC1C,eAAe;EAChB;;;;;;AAOH,SAAgB,yBAAyB,UAAkC;AACzE,QAAO,eACL,SAAS,cACT,SAAS,WACT,SAAS,QACT,SAAS,QACV;;;;;;;;AASH,SAAgB,uBAAuB,MAAkC;AACvE,QAAO;EACL,cAAc,KAAK;EACnB,WAAW,KAAK,WAAW;EAC3B,QAAQ,aAAa,KAAK;EAC1B,SAAS,cAAc,KAAK;EAC5B,oBAAoB,KAAK,0BAA0B;EACnD,eAAe;EAChB"}

package/out/zero-cache/src/services/replicator/schema/replication-state.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"replication-state.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/replication-state.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAGL,KAAK,UAAU,EAChB,MAAM,0CAA0C,CAAC;AAClD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AACzD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAC9D,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,2BAA2B,CAAC;AAG/D,OAAO,EAAC,wBAAwB,EAAC,MAAM,gBAAgB,CAAC;AAGxD,OAAO,EAAC,wBAAwB,EAAC,CAAC;AAElC,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;AAMzD,eAAO,MAAM,2BAA2B,iKAKvC,CAAC;~~AAwCF~~,QAAA,MAAM,uBAAuB;;;;EASxB,CAAC;AAEN,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,iCAAiC;;;;;EAclC,CAAC;AAEN,MAAM,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CAC/C,OAAO,iCAAiC,CACzC,CAAC;AAEF,QAAA,MAAM,sBAAsB;;aAE1B,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,QAAQ,EACZ,YAAY,EAAE,MAAM,EAAE,EACtB,SAAS,EAAE,MAAM,EACjB,kBAAkB,GAAE,UAAe,EACnC,YAAY,UAAO,~~QAqBpB~~;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAAC,EAAE,EAAE,QAAQ,QAExD;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,QAM5D;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,QAAQ;;;IAY9C;AAED,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,eAAe,GAAG,iBAAiB,CAQ3E;AAED,wBAAgB,8BAA8B,CAC5C,EAAE,EAAE,eAAe,GAClB,2BAA2B,CAS7B;AAED,wBAAgB,0BAA0B,CACxC,EAAE,EAAE,eAAe,EACnB,SAAS,EAAE,MAAM,~~QAGlB~~;AAED,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,eAAe,GAAG,gBAAgB,CAGzE"}
1	+ {"version":3,"file":"replication-state.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/schema/replication-state.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAGL,KAAK,UAAU,EAChB,MAAM,0CAA0C,CAAC;AAClD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AACzD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAC9D,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,2BAA2B,CAAC;AAG/D,OAAO,EAAC,wBAAwB,EAAC,MAAM,gBAAgB,CAAC;AAGxD,OAAO,EAAC,wBAAwB,EAAC,CAAC;AAElC,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;AAMzD,eAAO,MAAM,2BAA2B,iKAKvC,CAAC;AA0CF,QAAA,MAAM,uBAAuB;;;;EASxB,CAAC;AAEN,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,iCAAiC;;;;;EAclC,CAAC;AAEN,MAAM,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CAC/C,OAAO,iCAAiC,CACzC,CAAC;AAEF,QAAA,MAAM,sBAAsB;;aAE1B,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,QAAQ,EACZ,YAAY,EAAE,MAAM,EAAE,EACtB,SAAS,EAAE,MAAM,EACjB,kBAAkB,GAAE,UAAe,EACnC,YAAY,UAAO,QAoBpB;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAAC,EAAE,EAAE,QAAQ,QAExD;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,QAM5D;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,QAAQ;;;IAY9C;AAED,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,eAAe,GAAG,iBAAiB,CAQ3E;AAED,wBAAgB,8BAA8B,CAC5C,EAAE,EAAE,eAAe,GAClB,2BAA2B,CAS7B;AAED,wBAAgB,0BAA0B,CACxC,EAAE,EAAE,eAAe,EACnB,SAAS,EAAE,MAAM,QAQlB;AAED,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,eAAe,GAAG,gBAAgB,CAGzE"}

package/out/zero-cache/src/services/replicator/schema/replication-state.js CHANGED Viewed

@@ -17,7 +17,7 @@ var CREATE_RUNTIME_EVENTS_TABLE = `
     timestamp TEXT NOT NULL DEFAULT (current_timestamp)
   );
 `;
-var CREATE_REPLICATION_STATE_SCHEMA = "\n  CREATE TABLE \"_zero.replicationConfig\" (\n    replicaVersion TEXT NOT NULL,\n    publications TEXT NOT NULL,\n    initialSyncContext TEXT DEFAULT '{}',\n    lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n  \n  CREATE TABLE \"_zero.replicationState\" (\n    stateVersion TEXT NOT NULL,\n    lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n  " + CREATE_CHANGELOG_SCHEMA + CREATE_RUNTIME_EVENTS_TABLE + CREATE_COLUMN_METADATA_TABLE + CREATE_TABLE_METADATA_TABLE;
+var CREATE_REPLICATION_STATE_SCHEMA = "\n  CREATE TABLE \"_zero.replicationConfig\" (\n    replicaVersion TEXT NOT NULL,\n    publications TEXT NOT NULL,\n    initialSyncContext TEXT DEFAULT '{}',\n    lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n  \n  CREATE TABLE \"_zero.replicationState\" (\n    stateVersion TEXT NOT NULL,\n    writeTimeMs INTEGER,\n    lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n  " + CREATE_CHANGELOG_SCHEMA + CREATE_RUNTIME_EVENTS_TABLE + CREATE_COLUMN_METADATA_TABLE + CREATE_TABLE_METADATA_TABLE;
 var stringArray = valita_exports.array(valita_exports.string());
 var subscriptionStateSchema = valita_exports.object({
 	replicaVersion: valita_exports.string(),
@@ -45,7 +45,8 @@ function initReplicationState(db, publications, watermark, initialSyncContext =
        (replicaVersion, publications, initialSyncContext) VALUES (?, ?, ?)
     `).run(watermark, JSON.stringify(publications.sort()), stringify(initialSyncContext));
 	db.prepare(`
-    INSERT INTO "_zero.replicationState" (stateVersion) VALUES (?)
+    INSERT INTO "_zero.replicationState" (stateVersion, writeTimeMs)
+      VALUES (?, unixepoch('subsec') * 1000)
     `).run(watermark);
 	recordEvent(db, "sync");
 }
@@ -88,7 +89,9 @@ function getSubscriptionStateAndContext(db) {
     `), subscriptionStateAndContextSchema);
 }
 function updateReplicationWatermark(db, watermark) {
-	db.run(`UPDATE "_zero.replicationState" SET stateVersion=?`, watermark);
+	db.run(`
+    UPDATE "_zero.replicationState"
+      SET stateVersion=?, writeTimeMs=unixepoch('subsec') * 1000`, watermark);
 }
 function getReplicationState(db) {
 	return parse(db.get(`SELECT stateVersion FROM "_zero.replicationState"`), replicationStateSchema);

package/out/zero-cache/src/services/replicator/schema/replication-state.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"replication-state.js","names":[],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/replication-state.ts"],"sourcesContent":["/*\n Replication metadata, used for incremental view maintenance and catchup.\n \n These tables are created atomically in {@link setupReplicationTables}\n * after the logical replication handoff when initial data synchronization has completed.\n /\n\nimport {\n jsonObjectSchema,\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport as v from '../../../../../shared/src/valita.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../../db/statements.ts';\nimport {CREATE_CHANGELOG_SCHEMA} from './change-log.ts';\nimport {CREATE_COLUMN_METADATA_TABLE} from './column-metadata.ts';\nimport {ZERO_VERSION_COLUMN_NAME} from './constants.ts';\nimport {CREATE_TABLE_METADATA_TABLE} from './table-metadata.ts';\n\nexport {ZERO_VERSION_COLUMN_NAME};\n\nexport type RuntimeEvent = 'sync' \| 'upgrade' \| 'vacuum';\n\n// event : The RuntimeEvent. Only one row per event is tracked.\n// Inserting an event will REPLACE any row for the same event.\n// timestamp : SQLite timestamp string, e.g. \"2024-04-12 11:37:46\".\n// Append a `Z` when parsing with `new Date(...)`;\nexport const CREATE_RUNTIME_EVENTS_TABLE = `\n CREATE TABLE \"_zero.runtimeEvents\" (\n event TEXT PRIMARY KEY ON CONFLICT REPLACE,\n timestamp TEXT NOT NULL DEFAULT (current_timestamp)\n );\n`;\n\nconst CREATE_REPLICATION_STATE_SCHEMA =\n // replicaVersion : A value identifying the version at which the initial sync happened, i.e.\n // the version at which all rows were copied, and to `_0_version` was set.\n // This value is used to distinguish data from other replicas (e.g. if a\n // replica is reset or if there are ever multiple replicas).\n // publications : JSON stringified array of publication names\n // initialSyncContext : Metadata related to the context of when and how the replica was initially\n // synced. This corresponds with the same column stored in upstream and is\n // used for debugging replica version mismatches, which can arise from a number\n // of misconfigurations, such as dueling replication-managers, or restores of\n // stale litestream backups.\n // lock : Auto-magic column for enforcing single-row semantics.\n /sql/ `\n CREATE TABLE \"_zero.replicationConfig\" (\n replicaVersion TEXT NOT NULL,\n publications TEXT NOT NULL,\n initialSyncContext TEXT DEFAULT '{}',\n lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n ` +\n // stateVersion : The latest version replicated from upstream, starting with the initial\n // `replicaVersion` and moving forward to each subsequent commit watermark\n // (e.g. corresponding to a Postgres LSN). Versions are represented as\n // lexicographically sortable watermarks (e.g. LexiVersions).\n //\n `\n CREATE TABLE \"_zero.replicationState\" (\n stateVersion TEXT NOT NULL,\n lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n ` +\n CREATE_CHANGELOG_SCHEMA +\n CREATE_RUNTIME_EVENTS_TABLE +\n CREATE_COLUMN_METADATA_TABLE +\n CREATE_TABLE_METADATA_TABLE;\n\nconst stringArray = v.array(v.string());\n\nconst subscriptionStateSchema = v\n .object({\n replicaVersion: v.string(),\n publications: v.string(),\n watermark: v.string(),\n })\n .map(s => ({\n ...s,\n publications: v.parse(JSON.parse(s.publications), stringArray),\n }));\n\nexport type SubscriptionState = v.Infer<typeof subscriptionStateSchema>;\n\nconst subscriptionStateAndContextSchema = v\n .object({\n replicaVersion: v.string(),\n publications: v.string(),\n initialSyncContext: v.string(),\n watermark: v.string(),\n })\n .map(s => ({\n ...s,\n publications: v.parse(JSON.parse(s.publications), stringArray),\n initialSyncContext: v.parse(\n JSON.parse(s.initialSyncContext),\n jsonObjectSchema,\n ),\n }));\n\nexport type SubscriptionStateAndContext = v.Infer<\n typeof subscriptionStateAndContextSchema\n>;\n\nconst replicationStateSchema = v.object({\n stateVersion: v.string(),\n});\n\nexport type ReplicationState = v.Infer<typeof replicationStateSchema>;\n\nexport function initReplicationState(\n db: Database,\n publications: string[],\n watermark: string,\n initialSyncContext: JSONObject = {},\n createTables = true,\n) {\n if (createTables) {\n createReplicationStateTables(db);\n }\n db.prepare(\n `\n INSERT INTO \"_zero.replicationConfig\" \n (replicaVersion, publications, initialSyncContext) VALUES (?, ?, ?)\n `,\n ).run(\n watermark,\n JSON.stringify(publications.sort()),\n stringify(initialSyncContext),\n );\n db.prepare(\n `\n INSERT INTO \"_zero.replicationState\" (stateVersion) VALUES (?)\n ~~`,\n~~ ).run(watermark);\n recordEvent(db, 'sync');\n}\n\n/*\n Exposed as a separate function for the custom change source,\n * which needs the tables to be created in order to construct\n * ChangeProcessor before it knows the initial watermark.\n /\nexport function createReplicationStateTables(db: Database) {\n db.exec(CREATE_REPLICATION_STATE_SCHEMA);\n}\n\nexport function recordEvent(db: Database, event: RuntimeEvent) {\n db.prepare(\n `\n INSERT INTO \"_zero.runtimeEvents\" (event) VALUES (?) \n `,\n ).run(event);\n}\n\nexport function getAscendingEvents(db: Database) {\n const result = db\n .prepare(\n `SELECT event, timestamp FROM \"_zero.runtimeEvents\" \n ORDER BY timestamp ASC\n `,\n )\n .all<{event: string; timestamp: string}>();\n return result.map(({event, timestamp}) => ({\n event,\n timestamp: new Date(timestamp + 'Z'),\n }));\n}\n\nexport function getSubscriptionState(db: StatementRunner): SubscriptionState {\n const result = db.get(/sql/ `\n SELECT c.replicaVersion, c.publications, s.stateVersion as watermark\n FROM \"_zero.replicationConfig\" as c\n JOIN \"_zero.replicationState\" as s\n ON c.lock = s.lock\n `);\n return v.parse(result, subscriptionStateSchema);\n}\n\nexport function getSubscriptionStateAndContext(\n db: StatementRunner,\n): SubscriptionStateAndContext {\n const result = db.get(/sql*/ `\n SELECT c.replicaVersion, c.publications, c.initialSyncContext,\n s.stateVersion as watermark\n FROM \"_zero.replicationConfig\" as c\n JOIN \"_zero.replicationState\" as s\n ON c.lock = s.lock\n `);\n return v.parse(result, subscriptionStateAndContextSchema);\n}\n\nexport function updateReplicationWatermark(\n db: StatementRunner,\n watermark: string,\n) {\n db.run(`UPDATE \"_zero.replicationState\" SET stateVersion~~=?`,~~ watermark);\n}\n\nexport function getReplicationState(db: StatementRunner): ReplicationState {\n const result = db.get(`SELECT stateVersion FROM \"_zero.replicationState\"`);\n return v.parse(result, replicationStateSchema);\n}\n"],"mappings":";;;;;;;;;;;;;AA4BA,IAAa,8BAA8B;;;;;;AAO3C,IAAM,kCAYI,~~kXAmBR~~,0BACA,8BACA,+BACA;AAEF,IAAM,cAAc,eAAE,MAAM,eAAE,QAAQ,CAAC;AAEvC,IAAM,0BAA0B,eAC7B,OAAO;CACN,gBAAgB,eAAE,QAAQ;CAC1B,cAAc,eAAE,QAAQ;CACxB,WAAW,eAAE,QAAQ;CACtB,CAAC,CACD,KAAI,OAAM;CACT,GAAG;CACH,cAAc,MAAQ,KAAK,MAAM,EAAE,aAAa,EAAE,YAAY;CAC/D,EAAE;AAIL,IAAM,oCAAoC,eACvC,OAAO;CACN,gBAAgB,eAAE,QAAQ;CAC1B,cAAc,eAAE,QAAQ;CACxB,oBAAoB,eAAE,QAAQ;CAC9B,WAAW,eAAE,QAAQ;CACtB,CAAC,CACD,KAAI,OAAM;CACT,GAAG;CACH,cAAc,MAAQ,KAAK,MAAM,EAAE,aAAa,EAAE,YAAY;CAC9D,oBAAoB,MAClB,KAAK,MAAM,EAAE,mBAAmB,EAChC,iBACD;CACF,EAAE;AAML,IAAM,yBAAyB,eAAE,OAAO,EACtC,cAAc,eAAE,QAAQ,EACzB,CAAC;AAIF,SAAgB,qBACd,IACA,cACA,WACA,qBAAiC,EAAE,EACnC,eAAe,MACf;AACA,KAAI,aACF,8BAA6B,GAAG;AAElC,IAAG,QACD;;;MAID,CAAC,IACA,WACA,KAAK,UAAU,aAAa,MAAM,CAAC,EACnC,UAAU,mBAAmB,CAC9B;AACD,IAAG,~~QACD;;MAGD~~,CAAC,IAAI,UAAU;~~AAChB~~,aAAY,IAAI,OAAO;;;;;;;AAQzB,SAAgB,6BAA6B,IAAc;AACzD,IAAG,KAAK,gCAAgC;;AAG1C,SAAgB,YAAY,IAAc,OAAqB;AAC7D,IAAG,QACD;;MAGD,CAAC,IAAI,MAAM;;AAGd,SAAgB,mBAAmB,IAAc;AAQ/C,QAPe,GACZ,QACC;;MAGD,CACA,KAAyC,CAC9B,KAAK,EAAC,OAAO,iBAAgB;EACzC;EACA,2BAAW,IAAI,KAAK,YAAY,IAAI;EACrC,EAAE;;AAGL,SAAgB,qBAAqB,IAAwC;AAO3E,QAAO,MANQ,GAAG,IAAY;;;;;MAK1B,EACmB,wBAAwB;;AAGjD,SAAgB,+BACd,IAC6B;AAQ7B,QAAO,MAPQ,GAAG,IAAY;;;;;;MAM1B,EACmB,kCAAkC;;AAG3D,SAAgB,2BACd,IACA,WACA;AACA,IAAG,~~IAAI~~,~~sDAAsD,UAAU~~;;~~AAGzE~~,SAAgB,oBAAoB,IAAuC;AAEzE,QAAO,MADQ,GAAG,IAAI,oDAAoD,EACnD,uBAAuB"}
1	+ {"version":3,"file":"replication-state.js","names":[],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/replication-state.ts"],"sourcesContent":["/*\n Replication metadata, used for incremental view maintenance and catchup.\n \n These tables are created atomically in {@link setupReplicationTables}\n * after the logical replication handoff when initial data synchronization has completed.\n /\n\nimport {\n jsonObjectSchema,\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport as v from '../../../../../shared/src/valita.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../../db/statements.ts';\nimport {CREATE_CHANGELOG_SCHEMA} from './change-log.ts';\nimport {CREATE_COLUMN_METADATA_TABLE} from './column-metadata.ts';\nimport {ZERO_VERSION_COLUMN_NAME} from './constants.ts';\nimport {CREATE_TABLE_METADATA_TABLE} from './table-metadata.ts';\n\nexport {ZERO_VERSION_COLUMN_NAME};\n\nexport type RuntimeEvent = 'sync' \| 'upgrade' \| 'vacuum';\n\n// event : The RuntimeEvent. Only one row per event is tracked.\n// Inserting an event will REPLACE any row for the same event.\n// timestamp : SQLite timestamp string, e.g. \"2024-04-12 11:37:46\".\n// Append a `Z` when parsing with `new Date(...)`;\nexport const CREATE_RUNTIME_EVENTS_TABLE = `\n CREATE TABLE \"_zero.runtimeEvents\" (\n event TEXT PRIMARY KEY ON CONFLICT REPLACE,\n timestamp TEXT NOT NULL DEFAULT (current_timestamp)\n );\n`;\n\nconst CREATE_REPLICATION_STATE_SCHEMA =\n // replicaVersion : A value identifying the version at which the initial sync happened, i.e.\n // the version at which all rows were copied, and to `_0_version` was set.\n // This value is used to distinguish data from other replicas (e.g. if a\n // replica is reset or if there are ever multiple replicas).\n // publications : JSON stringified array of publication names\n // initialSyncContext : Metadata related to the context of when and how the replica was initially\n // synced. This corresponds with the same column stored in upstream and is\n // used for debugging replica version mismatches, which can arise from a number\n // of misconfigurations, such as dueling replication-managers, or restores of\n // stale litestream backups.\n // lock : Auto-magic column for enforcing single-row semantics.\n /sql/ `\n CREATE TABLE \"_zero.replicationConfig\" (\n replicaVersion TEXT NOT NULL,\n publications TEXT NOT NULL,\n initialSyncContext TEXT DEFAULT '{}',\n lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n ` +\n // stateVersion : The latest version replicated from upstream, starting with the initial\n // `replicaVersion` and moving forward to each subsequent commit watermark\n // (e.g. corresponding to a Postgres LSN). Versions are represented as\n // lexicographically sortable watermarks (e.g. LexiVersions).\n // writeTimeMs : The millisecond epoch at which this version was written to the replica.\n //\n /sql/ `\n CREATE TABLE \"_zero.replicationState\" (\n stateVersion TEXT NOT NULL,\n writeTimeMs INTEGER,\n lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n ` +\n CREATE_CHANGELOG_SCHEMA +\n CREATE_RUNTIME_EVENTS_TABLE +\n CREATE_COLUMN_METADATA_TABLE +\n CREATE_TABLE_METADATA_TABLE;\n\nconst stringArray = v.array(v.string());\n\nconst subscriptionStateSchema = v\n .object({\n replicaVersion: v.string(),\n publications: v.string(),\n watermark: v.string(),\n })\n .map(s => ({\n ...s,\n publications: v.parse(JSON.parse(s.publications), stringArray),\n }));\n\nexport type SubscriptionState = v.Infer<typeof subscriptionStateSchema>;\n\nconst subscriptionStateAndContextSchema = v\n .object({\n replicaVersion: v.string(),\n publications: v.string(),\n initialSyncContext: v.string(),\n watermark: v.string(),\n })\n .map(s => ({\n ...s,\n publications: v.parse(JSON.parse(s.publications), stringArray),\n initialSyncContext: v.parse(\n JSON.parse(s.initialSyncContext),\n jsonObjectSchema,\n ),\n }));\n\nexport type SubscriptionStateAndContext = v.Infer<\n typeof subscriptionStateAndContextSchema\n>;\n\nconst replicationStateSchema = v.object({\n stateVersion: v.string(),\n});\n\nexport type ReplicationState = v.Infer<typeof replicationStateSchema>;\n\nexport function initReplicationState(\n db: Database,\n publications: string[],\n watermark: string,\n initialSyncContext: JSONObject = {},\n createTables = true,\n) {\n if (createTables) {\n createReplicationStateTables(db);\n }\n db.prepare(\n `\n INSERT INTO \"_zero.replicationConfig\" \n (replicaVersion, publications, initialSyncContext) VALUES (?, ?, ?)\n `,\n ).run(\n watermark,\n JSON.stringify(publications.sort()),\n stringify(initialSyncContext),\n );\n db.prepare(/sql/ `\n INSERT INTO \"_zero.replicationState\" (stateVersion, writeTimeMs) \n VALUES (?, unixepoch('subsec') * 1000)\n `).run(watermark);\n recordEvent(db, 'sync');\n}\n\n/*\n Exposed as a separate function for the custom change source,\n * which needs the tables to be created in order to construct\n * ChangeProcessor before it knows the initial watermark.\n /\nexport function createReplicationStateTables(db: Database) {\n db.exec(CREATE_REPLICATION_STATE_SCHEMA);\n}\n\nexport function recordEvent(db: Database, event: RuntimeEvent) {\n db.prepare(\n `\n INSERT INTO \"_zero.runtimeEvents\" (event) VALUES (?) \n `,\n ).run(event);\n}\n\nexport function getAscendingEvents(db: Database) {\n const result = db\n .prepare(\n `SELECT event, timestamp FROM \"_zero.runtimeEvents\" \n ORDER BY timestamp ASC\n `,\n )\n .all<{event: string; timestamp: string}>();\n return result.map(({event, timestamp}) => ({\n event,\n timestamp: new Date(timestamp + 'Z'),\n }));\n}\n\nexport function getSubscriptionState(db: StatementRunner): SubscriptionState {\n const result = db.get(/sql/ `\n SELECT c.replicaVersion, c.publications, s.stateVersion as watermark\n FROM \"_zero.replicationConfig\" as c\n JOIN \"_zero.replicationState\" as s\n ON c.lock = s.lock\n `);\n return v.parse(result, subscriptionStateSchema);\n}\n\nexport function getSubscriptionStateAndContext(\n db: StatementRunner,\n): SubscriptionStateAndContext {\n const result = db.get(/sql/ `\n SELECT c.replicaVersion, c.publications, c.initialSyncContext,\n s.stateVersion as watermark\n FROM \"_zero.replicationConfig\" as c\n JOIN \"_zero.replicationState\" as s\n ON c.lock = s.lock\n `);\n return v.parse(result, subscriptionStateAndContextSchema);\n}\n\nexport function updateReplicationWatermark(\n db: StatementRunner,\n watermark: string,\n) {\n db.run(\n /sql/ `\n UPDATE \"_zero.replicationState\" \n SET stateVersion=?, writeTimeMs=unixepoch('subsec') 1000`,\n watermark,\n );\n}\n\nexport function getReplicationState(db: StatementRunner): ReplicationState {\n const result = db.get(`SELECT stateVersion FROM \"_zero.replicationState\"`);\n return v.parse(result, replicationStateSchema);\n}\n"],"mappings":";;;;;;;;;;;;;AA4BA,IAAa,8BAA8B;;;;;;AAO3C,IAAM,kCAYI,4YAqBR,0BACA,8BACA,+BACA;AAEF,IAAM,cAAc,eAAE,MAAM,eAAE,QAAQ,CAAC;AAEvC,IAAM,0BAA0B,eAC7B,OAAO;CACN,gBAAgB,eAAE,QAAQ;CAC1B,cAAc,eAAE,QAAQ;CACxB,WAAW,eAAE,QAAQ;CACtB,CAAC,CACD,KAAI,OAAM;CACT,GAAG;CACH,cAAc,MAAQ,KAAK,MAAM,EAAE,aAAa,EAAE,YAAY;CAC/D,EAAE;AAIL,IAAM,oCAAoC,eACvC,OAAO;CACN,gBAAgB,eAAE,QAAQ;CAC1B,cAAc,eAAE,QAAQ;CACxB,oBAAoB,eAAE,QAAQ;CAC9B,WAAW,eAAE,QAAQ;CACtB,CAAC,CACD,KAAI,OAAM;CACT,GAAG;CACH,cAAc,MAAQ,KAAK,MAAM,EAAE,aAAa,EAAE,YAAY;CAC9D,oBAAoB,MAClB,KAAK,MAAM,EAAE,mBAAmB,EAChC,iBACD;CACF,EAAE;AAML,IAAM,yBAAyB,eAAE,OAAO,EACtC,cAAc,eAAE,QAAQ,EACzB,CAAC;AAIF,SAAgB,qBACd,IACA,cACA,WACA,qBAAiC,EAAE,EACnC,eAAe,MACf;AACA,KAAI,aACF,8BAA6B,GAAG;AAElC,IAAG,QACD;;;MAID,CAAC,IACA,WACA,KAAK,UAAU,aAAa,MAAM,CAAC,EACnC,UAAU,mBAAmB,CAC9B;AACD,IAAG,QAAgB;;;MAGf,CAAC,IAAI,UAAU;AACnB,aAAY,IAAI,OAAO;;;;;;;AAQzB,SAAgB,6BAA6B,IAAc;AACzD,IAAG,KAAK,gCAAgC;;AAG1C,SAAgB,YAAY,IAAc,OAAqB;AAC7D,IAAG,QACD;;MAGD,CAAC,IAAI,MAAM;;AAGd,SAAgB,mBAAmB,IAAc;AAQ/C,QAPe,GACZ,QACC;;MAGD,CACA,KAAyC,CAC9B,KAAK,EAAC,OAAO,iBAAgB;EACzC;EACA,2BAAW,IAAI,KAAK,YAAY,IAAI;EACrC,EAAE;;AAGL,SAAgB,qBAAqB,IAAwC;AAO3E,QAAO,MANQ,GAAG,IAAY;;;;;MAK1B,EACmB,wBAAwB;;AAGjD,SAAgB,+BACd,IAC6B;AAQ7B,QAAO,MAPQ,GAAG,IAAY;;;;;;MAM1B,EACmB,kCAAkC;;AAG3D,SAAgB,2BACd,IACA,WACA;AACA,IAAG,IACO;;mEAGR,UACD;;AAGH,SAAgB,oBAAoB,IAAuC;AAEzE,QAAO,MADQ,GAAG,IAAI,oDAAoD,EACnD,uBAAuB"}

package/out/zero-cache/src/services/view-syncer/client-schema.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"client-schema.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/client-schema.ts"],"names":[],"mappings":"~~AAMA~~,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,gDAAgD,CAAC;AAGjF,OAAO,KAAK,EAAC,cAAc,EAAE,aAAa,EAAC,MAAM,mBAAmB,CAAC;AACrE,OAAO,EAA4B,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAG9E,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EACvC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,QAwGvC"}
1	+ {"version":3,"file":"client-schema.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/client-schema.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,gDAAgD,CAAC;AAGjF,OAAO,KAAK,EAAC,cAAc,EAAE,aAAa,EAAC,MAAM,mBAAmB,CAAC;AACrE,OAAO,EAA4B,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAG9E,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,YAAY,EAC1B,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EACvC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,QAwGvC"}

package/out/zero-cache/src/services/view-syncer/client-schema.js CHANGED Viewed

@@ -1,3 +1,4 @@
+import { toSorted } from "../../../../shared/src/iterables.js";
 import { must } from "../../../../shared/src/must.js";
 import { ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
 import { ProtocolError } from "../../../../zero-protocol/src/error.js";
@@ -14,7 +15,7 @@ function checkClientSchema(shardID, clientSchema, tableSpecs, fullTables) {
 	const errors = [];
 	const clientTables = new Set(Object.keys(clientSchema.tables));
 	const missingTables = difference(clientTables, tableSpecs);
-	for (const missing of [...missingTables].sort()) if (fullTables.has(missing)) errors.push(`The "${missing}" table is missing a primary key or non-null unique index and thus cannot be synced to the client`);
+	for (const missing of toSorted(missingTables)) if (fullTables.has(missing)) errors.push(`The "${missing}" table is missing a primary key or non-null unique index and thus cannot be synced to the client`);
 	else {
 		const app = appSchema(shardID) + ".";
 		const shard = upstreamSchema(shardID) + ".";
@@ -23,14 +24,14 @@ function checkClientSchema(shardID, clientSchema, tableSpecs, fullTables) {
 		errors.push(`The "${missing}" table does not exist or is not one of the replicated tables: ${syncedTables}.${schemaTip}`);
 	}
 	const tables = intersection(tableSpecs, clientTables);
-	for (const table of [...tables].sort()) {
+	for (const table of toSorted(tables)) {
 		const clientSpec = clientSchema.tables[table];
 		const serverSpec = must(tableSpecs.get(table));
 		const fullSpec = must(fullTables.get(table));
 		const clientColumns = new Set(Object.keys(clientSpec.columns));
 		const syncedColumns = new Set(Object.keys(serverSpec.zqlSpec));
 		const missingColumns = difference(clientColumns, syncedColumns);
-		for (const missing of [...missingColumns].sort()) if (fullSpec.columns[missing]) errors.push(`The "${table}"."${missing}" column cannot be synced because it is of an unsupported data type "${fullSpec.columns[missing].dataType}"`);
+		for (const missing of toSorted(missingColumns)) if (fullSpec.columns[missing]) errors.push(`The "${table}"."${missing}" column cannot be synced because it is of an unsupported data type "${fullSpec.columns[missing].dataType}"`);
 		else {
 			const columns = [...syncedColumns].filter((c) => c !== ZERO_VERSION_COLUMN_NAME).sort().map((c) => `"${c}"`).join(",");
 			errors.push(`The "${table}"."${missing}" column does not exist or is not one of the replicated columns: ${columns}.`);