npm - @rocicorp/zero - Versions diffs - 1.11.0-canary.13 → 1.11.0-canary.15 - Mend

@rocicorp/zero 1.11.0-canary.13 → 1.11.0-canary.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (252) hide show

package/out/zero-cache/src/services/plannerz.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"plannerz.js","names":[],"sources":["../../../../../zero-cache/src/services/plannerz.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {\n decodeSampleKinds,\n type SampleValueKind,\n} from '../../../zqlite/src/sqlite-stat4-sample.ts';\nimport type {NormalizedZeroConfig as ZeroConfig} from '../config/normalize.ts';\nimport {getServerVersion, isAdminPasswordValid} from '../config/zero-config.ts';\nimport {\n computeZqlSpecsFromLiteSpecs,\n listIndexes,\n listTables,\n} from '../db/lite-tables.ts';\nimport {StatementRunner} from '../db/statements.ts';\nimport {getReplicationState} from './replicator/schema/replication-state.ts';\n\n/**\n * How the statistics in the replica were gathered. zero-cache only ever runs\n * `PRAGMA optimize`, and always under an `analysis_limit`, which samples rather\n * than scanning. A replica that someone ran `ANALYZE` on by hand has better\n * statistics, which is visible in `sqlite_stat4` having rows at all.\n */\nconst APPROXIMATE_METHOD =\n 'PRAGMA optimize under an analysis_limit. Row counts and per-key averages ' +\n 'are sampled estimates, and sqlite_stat4 is not collected.';\nconst FULL_METHOD =\n 'ANALYZE, with no analysis limit. sqlite_stat4 has rows, so a full ANALYZE ' +\n 'ran against this replica at some point.';\n\n/**\n * A fixed glossary, sent with every bundle. The audience is a language model\n * that has the user's codebase but has never seen these statistics, and would\n * otherwise read a sampled estimate as an exact count.\n */\nconst ABOUT = {\n whatThisIs:\n 'Schema, indexes and SQLite statistics for the SQLite replica that ' +\n 'zero-cache queries. Use it to decide how to rewrite ZQL queries and ' +\n 'which indexes to add. It contains no row data.',\n stat1Format:\n 'sqlite_stat1.stat is \"<rows> <avg rows per distinct 1-column prefix> ' +\n '<... per 2-column prefix> ...\", parsed here into `rows` and ' +\n '`avgRowsPerPrefix`. NULLs count as ordinary values, so a sparse column ' +\n 'looks less selective than it is. Trailing flags such as \"unordered\", ' +\n '\"noskipscan\" and \"sz=N\" are kept in `flags`.',\n stat4Format:\n 'sqlite_stat4 is a per-index histogram. For each sample, `nEq` is the ' +\n 'rows equal to it, `nLt` the rows less than it, and `nDLt` the distinct ' +\n 'values less than it, one entry per key column. A large `nEq` next to a ' +\n 'small `nDLt` means a skewed column, where the average in stat1 hides ' +\n 'the common values. Sample values are never returned: `sample` reports ' +\n 'only each key column’s type.',\n howThePlannerUsesThis: [\n 'Cost comes from SQLite scanstatus estimates for the SQL each source ' +\n 'connection would run, so the statistics here are what the planner sees.',\n 'Join fanout comes from sqlite_stat4, else sqlite_stat1, else a default ' +\n 'of 3. With no stat4 (the usual case) fanout on a nullable foreign key ' +\n 'is overestimated, because stat1 counts NULLs.',\n 'The planner chooses per join whether to run it as a semi-join or to flip ' +\n 'it so the child drives. An index that matches the child’s ' +\n 'constraint plus ordering is what makes a flip cheap.',\n ],\n whatYouCanChange: [\n 'The ZQL query: `whereExists` vs `related`, the order of conditions, and ' +\n 'ordering that matches an existing index.',\n 'A `limit`, which lets the planner stop early.',\n 'Indexes: add them in Postgres, not on the replica. The replica copies ' +\n 'upstream indexes, so a new index reaches it through replication.',\n 'The planner flags in `server.planner`.',\n ],\n caveats: [\n 'Row counts are estimates from the statistics, not COUNT(*). This ' +\n 'endpoint never scans a table.',\n 'Statistics are as of the last time they were gathered, which SQLite does ' +\n 'not record. A table whose size changed a lot since then can be far off.',\n 'Tables in `tables` with `syncable: false` cannot be queried by clients: ' +\n 'they lack a primary key or unique index, or every column has a type ' +\n 'ZQL does not support.',\n ],\n nextStep:\n 'POST an AST to /plannerz/analyze to see the plan the planner picks for ' +\n 'one query, with the SQLite query plans it generates.',\n} as const;\n\nexport type Stat1 = {\n /** The raw `sqlite_stat1.stat` text, in case a flag is not parsed here. */\n raw: string;\n /** Estimated rows in the table. */\n rows: number;\n /**\n * Average rows per distinct key prefix: the first entry is for a 1-column\n * prefix, the second for 2 columns, and so on.\n */\n avgRowsPerPrefix: number[];\n /** Non-numeric tokens, e.g. `unordered`, `noskipscan`, `sz=N`. */\n flags: string[];\n};\n\n/** A `sqlite_stat4` sample with its values replaced by their types. */\nexport type Stat4Sample = {\n nEq: number[];\n nLt: number[];\n nDLt: number[];\n /** One entry per key column, plus the trailing rowid. */\n sample: SampleValueKind[];\n};\n\n/**\n * A digest of an index's `sqlite_stat4` histogram, one entry per key prefix\n * depth. This is what the default response carries: a full histogram runs to\n * hundreds of samples per index, which is a lot of context to spend on data an\n * LLM would only summarize anyway.\n */\nexport type Stat4PrefixSummary = {\n /** Rows equal to the most common sampled key at this depth. */\n maxRowsPerKey: number;\n /** Median rows per sampled key at this depth. */\n medianRowsPerKey: number;\n /**\n * Lower bound on distinct keys at this depth, from the last sample's count of\n * distinct values less than it.\n */\n estimatedDistinctKeys: number;\n};\n\nexport type Stat4Summary = {\n samples: number;\n /** The type of each key column, and of the trailing rowid. */\n sampleKinds: SampleValueKind[];\n /** One entry per key prefix depth: 1 column, then 2, and so on. */\n perPrefix: Stat4PrefixSummary[];\n};\n\nexport type IndexStats = {\n name: string;\n columns: {name: string; dir: 'ASC' | 'DESC'}[];\n unique: boolean;\n partial: boolean;\n stat1?: Stat1 | undefined;\n stat4?: Stat4Summary | undefined;\n /** The full histogram, only with `?stat4=full`. */\n stat4Samples?: Stat4Sample[] | undefined;\n};\n\nexport type ColumnStats = {\n name: string;\n dataType: string;\n nullable: boolean;\n /** The ZQL type, or `null` for a column clients cannot see. */\n zqlType: string | null;\n backfilling?: true | undefined;\n};\n\nexport type TableStats = {\n name: string;\n columns: ColumnStats[];\n primaryKey: string[] | undefined;\n estimatedRows: number | undefined;\n syncable: boolean;\n indexes: IndexStats[];\n};\n\nexport type PlannerzBundle = {\n about: typeof ABOUT;\n server: {\n zeroVersion: string;\n sqliteVersion: string;\n replicaWatermark: string | undefined;\n generatedAt: string;\n planner: {\n enableQueryPlanner: boolean;\n enableCorrelatedPredicatePushdown: boolean;\n enablePlannerAwarePushdown: boolean;\n };\n };\n statsQuality: {\n stat1Present: boolean;\n stat4Rows: number;\n method: string;\n tablesWithoutStats: string[];\n };\n tables: TableStats[];\n};\n\n/**\n * Parses a `sqlite_stat1.stat` value.\n *\n * @visibleForTesting\n */\nexport function parseStat1(raw: string): Stat1 {\n const tokens = raw.trim().split(SEPARATOR).filter(Boolean);\n const numbers: number[] = [];\n const flags: string[] = [];\n for (const token of tokens) {\n // Only leading tokens are counts; anything else is a flag, even if it\n // parses as a number (e.g. a future \"sz=N\" spelled differently).\n if (flags.length === 0 && COUNT.test(token)) {\n numbers.push(parseInt(token, 10));\n } else {\n flags.push(token);\n }\n }\n return {\n raw,\n rows: numbers[0] ?? 0,\n avgRowsPerPrefix: numbers.slice(1),\n flags,\n };\n}\n\n/** Whitespace between the counts in a `sqlite_stat1` or `sqlite_stat4` value. */\nconst SEPARATOR = /\\s+/;\n/** A count, as opposed to a trailing flag such as `unordered` or `sz=N`. */\nconst COUNT = /^\\d+$/;\n\ntype Stat1Row = {tbl: string; idx: string | null; stat: string};\ntype Stat4Row = {\n tbl: string;\n idx: string;\n neq: string;\n nlt: string;\n ndlt: string;\n sample: Buffer;\n};\n\nfunction tableExists(db: Database, name: string): boolean {\n return (\n db\n .prepare(\n `SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`,\n )\n .all(name).length > 0\n );\n}\n\nfunction parseCounts(raw: string): number[] {\n return raw\n .trim()\n .split(SEPARATOR)\n .filter(Boolean)\n .map(n => parseInt(n, 10));\n}\n\n/**\n * Builds the `/plannerz` bundle from the replica's catalog, `sqlite_stat*`\n * tables and the server config. No user table is read, so the cost does not\n * depend on how much data the replica holds.\n *\n * @visibleForTesting\n */\nexport function buildPlannerzBundle(\n lc: LogContext,\n config: ZeroConfig,\n opts: BundleOptions = {},\n): PlannerzBundle {\n using db = new Database(lc, config.replica.file, {readonly: true});\n return buildPlannerzBundleFromDB(lc, db, config, opts);\n}\n\nexport type BundleOptions = {\n /** Include the full `sqlite_stat4` histogram, not just the digest. */\n fullStat4?: boolean | undefined;\n};\n\n/** @visibleForTesting */\nexport function buildPlannerzBundleFromDB(\n lc: LogContext,\n db: Database,\n config: ZeroConfig,\n opts: BundleOptions = {},\n): PlannerzBundle {\n const tables = listReplicaTables(lc, db);\n const indexes = listIndexes(db);\n const zqlSpecs = computeZqlSpecsFromLiteSpecs(\n tables,\n indexes,\n {includeBackfillingColumns: false},\n undefined,\n undefined,\n lc,\n );\n\n const stat1Present = tableExists(db, 'sqlite_stat1');\n const stat1ByTableAndIndex = new Map<string, Stat1>();\n const statsByTable = new Map<string, Stat1>();\n if (stat1Present) {\n for (const {tbl, idx, stat} of db\n .prepare(`SELECT tbl, idx, stat FROM sqlite_stat1`)\n .all() as Stat1Row[]) {\n const parsed = parseStat1(stat);\n if (idx !== null) {\n stat1ByTableAndIndex.set(`${tbl}.${idx}`, parsed);\n }\n // Every row for a table starts with the table's row count. A row with a\n // null `idx` is written for a table with no indexes, and is preferred\n // because it is about the table itself.\n if (idx === null || !statsByTable.has(tbl)) {\n statsByTable.set(tbl, parsed);\n }\n }\n }\n\n const stat4Present = tableExists(db, 'sqlite_stat4');\n const stat4ByTableAndIndex = new Map<string, Stat4Sample[]>();\n let stat4Rows = 0;\n if (stat4Present) {\n for (const {tbl, idx, neq, nlt, ndlt, sample} of db\n .prepare(`SELECT tbl, idx, neq, nlt, ndlt, sample FROM sqlite_stat4`)\n .all() as Stat4Row[]) {\n stat4Rows++;\n const samples = stat4ByTableAndIndex.get(`${tbl}.${idx}`) ?? [];\n samples.push({\n nEq: parseCounts(neq),\n nLt: parseCounts(nlt),\n nDLt: parseCounts(ndlt),\n // Values are deliberately dropped here: only the types are reported.\n sample: decodeSampleKinds(sample),\n });\n stat4ByTableAndIndex.set(`${tbl}.${idx}`, samples);\n }\n }\n\n const indexesByTable = new Map<string, IndexStats[]>();\n for (const index of indexes) {\n const key = `${index.tableName}.${index.name}`;\n const stat4 = stat4ByTableAndIndex.get(key);\n const forTable = indexesByTable.get(index.tableName) ?? [];\n forTable.push({\n name: index.name,\n columns: Object.entries(index.columns).map(([name, dir]) => ({\n name,\n dir,\n })),\n unique: index.unique,\n partial: index.partial === true,\n stat1: stat1ByTableAndIndex.get(key),\n stat4: stat4?.length ? summarizeStat4(stat4) : undefined,\n stat4Samples: opts.fullStat4 && stat4?.length ? stat4 : undefined,\n });\n indexesByTable.set(index.tableName, forTable);\n }\n\n const tablesWithoutStats: string[] = [];\n const tableStats = tables.map(table => {\n const zqlSpec = zqlSpecs.get(table.name)?.zqlSpec;\n if (!statsByTable.has(table.name)) {\n tablesWithoutStats.push(table.name);\n }\n const backfilling = new Set(table.backfilling ?? []);\n return {\n name: table.name,\n columns: Object.entries(table.columns).map(([name, spec]) => ({\n name,\n dataType: spec.dataType,\n nullable: !spec.notNull,\n zqlType: zqlSpec?.[name]?.type ?? null,\n ...(backfilling.has(name) ? {backfilling: true as const} : {}),\n })),\n primaryKey: table.primaryKey ? [...table.primaryKey] : undefined,\n estimatedRows: statsByTable.get(table.name)?.rows,\n syncable: zqlSpec !== undefined,\n indexes: indexesByTable.get(table.name) ?? [],\n };\n });\n\n return {\n about: ABOUT,\n server: {\n zeroVersion: getServerVersion(config),\n sqliteVersion: sqliteVersion(db),\n replicaWatermark: replicaWatermark(lc, db),\n generatedAt: new Date().toISOString(),\n planner: {\n enableQueryPlanner: config.enableQueryPlanner,\n // Both default to on, matching services/analyze.ts.\n enableCorrelatedPredicatePushdown:\n config.enableCorrelatedPredicatePushdown !== false,\n enablePlannerAwarePushdown: config.enablePlannerAwarePushdown !== false,\n },\n },\n statsQuality: {\n stat1Present,\n stat4Rows,\n method: stat4Rows > 0 ? FULL_METHOD : APPROXIMATE_METHOD,\n tablesWithoutStats,\n },\n tables: tableStats,\n };\n}\n\n/**\n * Lists the replica's tables, tolerating a replica that has no metadata tables\n * yet. `listTables` reads `_zero.tableMetadata` for backfill status, which does\n * not exist until initial sync has created it. Reporting the schema without\n * backfill status beats failing the whole request.\n */\nfunction listReplicaTables(lc: LogContext, db: Database) {\n try {\n return listTables(db);\n } catch (e) {\n lc.debug?.('reading table metadata failed; listing tables without it', e);\n return listTables(db, true, false);\n }\n}\n\n/**\n * Digests an index's stat4 samples.\n *\n * Note that `nLt` and `nDLt` are stored as text, so SQL cannot order the\n * samples by them numerically. This works off the whole set rather than\n * relying on the samples' order: the highest `nDLt` is a lower bound on the\n * distinct keys, since the sample holding it has that many distinct keys below\n * it plus itself.\n *\n * @visibleForTesting\n */\nexport function summarizeStat4(samples: Stat4Sample[]): Stat4Summary {\n const depth = Math.max(...samples.map(s => s.nEq.length));\n const perPrefix: Stat4PrefixSummary[] = [];\n for (let d = 0; d < depth; d++) {\n const counts = samples\n .map(s => s.nEq[d])\n .filter(n => n !== undefined)\n .sort((a, b) => a - b);\n const maxDistinctBelow = Math.max(\n ...samples.map(s => s.nDLt[d]).filter(n => n !== undefined),\n );\n perPrefix.push({\n maxRowsPerKey: counts.at(-1)!,\n medianRowsPerKey: counts[Math.floor((counts.length - 1) / 2)],\n estimatedDistinctKeys: maxDistinctBelow + 1,\n });\n }\n return {\n samples: samples.length,\n // Every sample of an index describes the same columns.\n sampleKinds: samples[0].sample,\n perPrefix,\n };\n}\n\nfunction sqliteVersion(db: Database): string {\n const [{version}] = db\n .prepare(`SELECT sqlite_version() AS version`)\n .all() as {version: string}[];\n return version;\n}\n\nfunction replicaWatermark(lc: LogContext, db: Database): string | undefined {\n try {\n return getReplicationState(new StatementRunner(db)).stateVersion;\n } catch (e) {\n // A replica that has not finished initial sync has no replication state.\n lc.debug?.('could not read replication state', e);\n return undefined;\n }\n}\n\n/**\n * Serves the replica's schema, indexes and SQLite statistics for consumption by\n * a language model, so that it can advise on query rewrites and indexes.\n *\n * Row data is never returned. See `designs/003_replica_stats_endpoint.md`.\n *\n * HTTP query parameters:\n * * `pretty`: indents the JSON\n * * `stat4=full`: includes the full sqlite_stat4 histogram, which is large\n */\nexport async function handlePlannerzRequest(\n lc: LogContext,\n config: ZeroConfig,\n req: FastifyRequest,\n res: FastifyReply,\n) {\n const credentials = auth(req);\n if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n void res\n .code(401)\n .header('WWW-Authenticate', 'Basic realm=\"Plannerz Protected Area\"')\n .send('Unauthorized');\n return;\n }\n\n const query = req.query as Record<string, unknown>;\n const indent = query.pretty !== undefined ? 2 : undefined;\n let bundle: PlannerzBundle;\n try {\n bundle = buildPlannerzBundle(lc, config, {\n fullStat4: query.stat4 === 'full',\n });\n } catch (e) {\n lc.error?.('error building plannerz bundle', e);\n await res.code(500).send({error: String(e)});\n return;\n }\n await res\n .header('Content-Type', 'application/json')\n .send(JSON.stringify(bundle, null, indent));\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwBA,IAAM,qBACJ;AAEF,IAAM,cACJ;;;;;;AAQF,IAAM,QAAQ;CACZ,YACE;CAGF,aACE;CAKF,aACE;CAMF,uBAAuB;EACrB;EAEA;EAGA;CAGF;CACA,kBAAkB;EAChB;EAEA;EACA;EAEA;CACF;CACA,SAAS;EACP;EAEA;EAEA;CAGF;CACA,UACE;AAEJ;;;;;;AA2GA,SAAgB,WAAW,KAAoB;CAC7C,MAAM,SAAS,IAAI,KAAK,EAAE,MAAM,SAAS,EAAE,OAAO,OAAO;CACzD,MAAM,UAAoB,CAAC;CAC3B,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,SAAS,QAGlB,IAAI,MAAM,WAAW,KAAK,MAAM,KAAK,KAAK,GACxC,QAAQ,KAAK,SAAS,OAAO,EAAE,CAAC;MAEhC,MAAM,KAAK,KAAK;CAGpB,OAAO;EACL;EACA,MAAM,QAAQ,MAAM;EACpB,kBAAkB,QAAQ,MAAM,CAAC;EACjC;CACF;AACF;;AAGA,IAAM,YAAY;;AAElB,IAAM,QAAQ;AAYd,SAAS,YAAY,IAAc,MAAuB;CACxD,OACE,GACG,QACC,kEACF,EACC,IAAI,IAAI,EAAE,SAAS;AAE1B;AAEA,SAAS,YAAY,KAAuB;CAC1C,OAAO,IACJ,KAAK,EACL,MAAM,SAAS,EACf,OAAO,OAAO,EACd,KAAI,MAAK,SAAS,GAAG,EAAE,CAAC;AAC7B;;;;;;;;AASA,SAAgB,oBACd,IACA,QACA,OAAsB,CAAC,GACP;;;EAEhB,OAAO,0BAA0B,IAAA,YAAA,EADtB,IAAI,SAAS,IAAI,OAAO,QAAQ,MAAM,EAAC,UAAU,KAAI,CAAC,CAC5B,GAAI,QAAQ,IAAI;;;;;;AACvD;;AAQA,SAAgB,0BACd,IACA,IACA,QACA,OAAsB,CAAC,GACP;CAChB,MAAM,SAAS,kBAAkB,IAAI,EAAE;CACvC,MAAM,UAAU,YAAY,EAAE;CAC9B,MAAM,WAAW,6BACf,QACA,SACA,EAAC,2BAA2B,MAAK,GACjC,KAAA,GACA,KAAA,GACA,EACF;CAEA,MAAM,eAAe,YAAY,IAAI,cAAc;CACnD,MAAM,uCAAuB,IAAI,IAAmB;CACpD,MAAM,+BAAe,IAAI,IAAmB;CAC5C,IAAI,cACF,KAAK,MAAM,EAAC,KAAK,KAAK,UAAS,GAC5B,QAAQ,yCAAyC,EACjD,IAAI,GAAiB;EACtB,MAAM,SAAS,WAAW,IAAI;EAC9B,IAAI,QAAQ,MACV,qBAAqB,IAAI,GAAG,IAAI,GAAG,OAAO,MAAM;EAKlD,IAAI,QAAQ,QAAQ,CAAC,aAAa,IAAI,GAAG,GACvC,aAAa,IAAI,KAAK,MAAM;CAEhC;CAGF,MAAM,eAAe,YAAY,IAAI,cAAc;CACnD,MAAM,uCAAuB,IAAI,IAA2B;CAC5D,IAAI,YAAY;CAChB,IAAI,cACF,KAAK,MAAM,EAAC,KAAK,KAAK,KAAK,KAAK,MAAM,YAAW,GAC9C,QAAQ,2DAA2D,EACnE,IAAI,GAAiB;EACtB;EACA,MAAM,UAAU,qBAAqB,IAAI,GAAG,IAAI,GAAG,KAAK,KAAK,CAAC;EAC9D,QAAQ,KAAK;GACX,KAAK,YAAY,GAAG;GACpB,KAAK,YAAY,GAAG;GACpB,MAAM,YAAY,IAAI;GAEtB,QAAQ,kBAAkB,MAAM;EAClC,CAAC;EACD,qBAAqB,IAAI,GAAG,IAAI,GAAG,OAAO,OAAO;CACnD;CAGF,MAAM,iCAAiB,IAAI,IAA0B;CACrD,KAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,MAAM,GAAG,MAAM,UAAU,GAAG,MAAM;EACxC,MAAM,QAAQ,qBAAqB,IAAI,GAAG;EAC1C,MAAM,WAAW,eAAe,IAAI,MAAM,SAAS,KAAK,CAAC;EACzD,SAAS,KAAK;GACZ,MAAM,MAAM;GACZ,SAAS,OAAO,QAAQ,MAAM,OAAO,EAAE,KAAK,CAAC,MAAM,UAAU;IAC3D;IACA;GACF,EAAE;GACF,QAAQ,MAAM;GACd,SAAS,MAAM,YAAY;GAC3B,OAAO,qBAAqB,IAAI,GAAG;GACnC,OAAO,OAAO,SAAS,eAAe,KAAK,IAAI,KAAA;GAC/C,cAAc,KAAK,aAAa,OAAO,SAAS,QAAQ,KAAA;EAC1D,CAAC;EACD,eAAe,IAAI,MAAM,WAAW,QAAQ;CAC9C;CAEA,MAAM,qBAA+B,CAAC;CACtC,MAAM,aAAa,OAAO,KAAI,UAAS;EACrC,MAAM,UAAU,SAAS,IAAI,MAAM,IAAI,GAAG;EAC1C,IAAI,CAAC,aAAa,IAAI,MAAM,IAAI,GAC9B,mBAAmB,KAAK,MAAM,IAAI;EAEpC,MAAM,cAAc,IAAI,IAAI,MAAM,eAAe,CAAC,CAAC;EACnD,OAAO;GACL,MAAM,MAAM;GACZ,SAAS,OAAO,QAAQ,MAAM,OAAO,EAAE,KAAK,CAAC,MAAM,WAAW;IAC5D;IACA,UAAU,KAAK;IACf,UAAU,CAAC,KAAK;IAChB,SAAS,UAAU,OAAO,QAAQ;IAClC,GAAI,YAAY,IAAI,IAAI,IAAI,EAAC,aAAa,KAAa,IAAI,CAAC;GAC9D,EAAE;GACF,YAAY,MAAM,aAAa,CAAC,GAAG,MAAM,UAAU,IAAI,KAAA;GACvD,eAAe,aAAa,IAAI,MAAM,IAAI,GAAG;GAC7C,UAAU,YAAY,KAAA;GACtB,SAAS,eAAe,IAAI,MAAM,IAAI,KAAK,CAAC;EAC9C;CACF,CAAC;CAED,OAAO;EACL,OAAO;EACP,QAAQ;GACN,aAAa,iBAAiB,MAAM;GACpC,eAAe,cAAc,EAAE;GAC/B,kBAAkB,iBAAiB,IAAI,EAAE;GACzC,8BAAa,IAAI,KAAK,GAAE,YAAY;GACpC,SAAS;IACP,oBAAoB,OAAO;IAE3B,mCACE,OAAO,sCAAsC;IAC/C,4BAA4B,OAAO,+BAA+B;GACpE;EACF;EACA,cAAc;GACZ;GACA;GACA,QAAQ,YAAY,IAAI,cAAc;GACtC;EACF;EACA,QAAQ;CACV;AACF;;;;;;;AAQA,SAAS,kBAAkB,IAAgB,IAAc;CACvD,IAAI;EACF,OAAO,WAAW,EAAE;CACtB,SAAS,GAAG;EACV,GAAG,QAAQ,4DAA4D,CAAC;EACxE,OAAO,WAAW,IAAI,MAAM,KAAK;CACnC;AACF;;;;;;;;;;;;AAaA,SAAgB,eAAe,SAAsC;CACnE,MAAM,QAAQ,KAAK,IAAI,GAAG,QAAQ,KAAI,MAAK,EAAE,IAAI,MAAM,CAAC;CACxD,MAAM,YAAkC,CAAC;CACzC,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;EAC9B,MAAM,SAAS,QACZ,KAAI,MAAK,EAAE,IAAI,EAAE,EACjB,QAAO,MAAK,MAAM,KAAA,CAAS,EAC3B,MAAM,GAAG,MAAM,IAAI,CAAC;EACvB,MAAM,mBAAmB,KAAK,IAC5B,GAAG,QAAQ,KAAI,MAAK,EAAE,KAAK,EAAE,EAAE,QAAO,MAAK,MAAM,KAAA,CAAS,CAC5D;EACA,UAAU,KAAK;GACb,eAAe,OAAO,GAAG,EAAE;GAC3B,kBAAkB,OAAO,KAAK,OAAO,OAAO,SAAS,KAAK,CAAC;GAC3D,uBAAuB,mBAAmB;EAC5C,CAAC;CACH;CACA,OAAO;EACL,SAAS,QAAQ;EAEjB,aAAa,QAAQ,GAAG;EACxB;CACF;AACF;AAEA,SAAS,cAAc,IAAsB;CAC3C,MAAM,CAAC,EAAC,aAAY,GACjB,QAAQ,oCAAoC,EAC5C,IAAI;CACP,OAAO;AACT;AAEA,SAAS,iBAAiB,IAAgB,IAAkC;CAC1E,IAAI;EACF,OAAO,oBAAoB,IAAI,gBAAgB,EAAE,CAAC,EAAE;CACtD,SAAS,GAAG;EAEV,GAAG,QAAQ,oCAAoC,CAAC;EAChD;CACF;AACF;;;;;;;;;;;AAYA,eAAsB,sBACpB,IACA,QACA,KACA,KACA;CAEA,IAAI,CAAC,qBAAqB,IAAI,QADV,KAAK,GACa,GAAa,IAAI,GAAG;EACxD,IACG,KAAK,GAAG,EACR,OAAO,oBAAoB,yCAAuC,EAClE,KAAK,cAAc;EACtB;CACF;CAEA,MAAM,QAAQ,IAAI;CAClB,MAAM,SAAS,MAAM,WAAW,KAAA,IAAY,IAAI,KAAA;CAChD,IAAI;CACJ,IAAI;EACF,SAAS,oBAAoB,IAAI,QAAQ,EACvC,WAAW,MAAM,UAAU,OAC7B,CAAC;CACH,SAAS,GAAG;EACV,GAAG,QAAQ,kCAAkC,CAAC;EAC9C,MAAM,IAAI,KAAK,GAAG,EAAE,KAAK,EAAC,OAAO,OAAO,CAAC,EAAC,CAAC;EAC3C;CACF;CACA,MAAM,IACH,OAAO,gBAAgB,kBAAkB,EACzC,KAAK,KAAK,UAAU,QAAQ,MAAM,MAAM,CAAC;AAC9C"}

package/out/zero-cache/src/services/replicator/change-log-cookies.js CHANGED Viewed

@@ -360,7 +360,7 @@ function backfillRequestsFrom(cookies) {
 			};
 			requests.push(curr);
 		}
-		curr.columns[column] = backfill;
+		curr.columns[column] = { id: backfill };
 	}
 	return parse(requests, backfillRequestsSchema);
 }

package/out/zero-cache/src/services/replicator/change-log-cookies.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-log-cookies.js","names":["#upsertMetadata","#upsertBackfill","#renameMetadataTable","#renameBackfillTable","#deleteMetadataTable","#deleteBackfillTable","#renameBackfillColumn","#deleteBackfillColumn","#run"],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-cookies.ts"],"sourcesContent":["/*\n The change log's second job.\n \n A change log is a buffer and a cookie jar. The buffer catches a\n * reconnecting subscriber up to the head; the cookie jar holds the backfill\n * progress state that the change-streamer hands the change-source on every\n * `startStream`, so that the change source never has to persist backfill state\n * upstream. See `change-log-db.ts` for the buffer.\n \n There are two cookies, mirroring the two `cdc` tables the Postgres change log\n * has always carried (`change-streamer/schema/tables.ts`):\n \n - table metadata, the most current {@link TableMetadata} of a table, which\n * a {@link BackfillRequest} has to carry; and\n * - backfilling columns, the opaque {@link BackfillID} of every column whose\n * backfill has not completed.\n \n Both are a fold over the change stream: the cookie set is only meaningful\n * paired with the watermark it was folded up to. {@link cookieOps} is that fold,\n * expressed transport-free so that the Postgres change log and the SQLite change\n * log cannot drift from each other — they must agree on all eight transitions\n * forever, and the failure mode if they disagree is a backfill that is silently\n * never re-requested. The two interpreters are {@link ChangeLogCookieWriter}\n * here and `Storer.#trackBackfillMetadata()` in\n * `change-streamer/storer.ts`.\n \n The tables live in the change-log database rather than in the replica: the\n * replicator is a subscriber downstream of the change-streamer's write loop,\n * so its state version trails the log's head, and pairing its cookies with the\n * log's resume watermark would lose every backfill that completed in between.\n * The replica's own copies (`_zero.tableMetadata`, `_zero.column_metadata`) are\n * the seed source; these are the working set.\n /\n\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport as v from '../../../../shared/src/valita.ts';\nimport type {Database, Statement} from '../../../../zqlite/src/db.ts';\nimport type {\n BackfillID,\n Identifier,\n SchemaChange,\n TableMetadata,\n} from '../change-source/protocol/current/data.ts';\nimport {\n backfillRequestSchema,\n type BackfillRequest,\n} from '../change-source/protocol/current/upstream.ts';\n\nexport const CHANGE_LOG_TABLE_METADATA_TABLE = '_zero.changeLogTableMetadata';\nexport const CHANGE_LOG_BACKFILLING_TABLE = '_zero.changeLogBackfilling';\n\n// The names follow the file's existing convention, beside\n// \"_zero.changeLogStream\" and \"_zero.changeLogMeta\".\n//\n// `metadata` and `backfill` hold the JSON that the `cdc` tables hold as JSONB.\n// SQLite has no JSONB, and nothing here queries into the documents: they are\n// stored to be handed back to the change source verbatim.\nexport const CREATE_CHANGE_LOG_COOKIE_SCHEMA = /sql/ `\n CREATE TABLE \"${CHANGE_LOG_TABLE_METADATA_TABLE}\" (\n \"schema\" TEXT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"metadata\" TEXT NOT NULL,\n PRIMARY KEY (\"schema\", \"table\")\n );\n\n CREATE TABLE \"${CHANGE_LOG_BACKFILLING_TABLE}\" (\n \"schema\" TEXT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"column\" TEXT NOT NULL,\n \"backfill\" TEXT NOT NULL,\n PRIMARY KEY (\"schema\", \"table\", \"column\")\n );\n`;\n\nexport const DROP_CHANGE_LOG_COOKIE_TABLES = /sql/ `\n DROP TABLE IF EXISTS \"${CHANGE_LOG_TABLE_METADATA_TABLE}\";\n DROP TABLE IF EXISTS \"${CHANGE_LOG_BACKFILLING_TABLE}\";\n`;\n\nexport type TableMetadataCookie = {\n readonly schema: string;\n readonly table: string;\n readonly metadata: TableMetadata;\n};\n\nexport type BackfillCookie = {\n readonly schema: string;\n readonly table: string;\n readonly column: string;\n readonly backfill: BackfillID;\n};\n\n/*\n The whole cookie set, as of the log's head. Both lists are ordered by their\n * primary key so that two stores' sets can be compared directly.\n /\nexport type CookieSet = {\n readonly tableMetadata: readonly TableMetadataCookie[];\n readonly backfilling: readonly BackfillCookie[];\n};\n\nexport const EMPTY_COOKIE_SET: CookieSet = {\n tableMetadata: [],\n backfilling: [],\n};\n\n/\n A transport-free description of a cookie mutation. This is the whole of what\n * a {@link SchemaChange} means to the cookie jar; everything else about the\n * change belongs to the buffer.\n /\nexport type CookieOp =\n \| {op: 'upsert-metadata'; table: Identifier; metadata: TableMetadata}\n \| {\n op: 'upsert-backfill';\n table: Identifier;\n column: string;\n backfill: BackfillID;\n }\n \| {op: 'rename-table'; old: Identifier; new: Identifier}\n \| {op: 'drop-table'; table: Identifier}\n \| {op: 'rename-column'; table: Identifier; old: string; new: string}\n \| {op: 'drop-column'; table: Identifier; column: string}\n \| {op: 'complete-backfill'; table: Identifier; columns: readonly string[]};\n\nexport type CookieOpTag = CookieOp['op'];\n\n/\n The fold, and the only place either store decides what a schema change means\n * for the cookie set. Both interpreters are mechanical translations of what this\n * returns, in the order it returns them.\n \n Changes that carry no cookie state — index changes, and the `create-table` /\n * `add-column` variants whose optional `metadata` and `backfill` fields are\n * absent — produce no ops.\n /\nexport function cookieOps(change: SchemaChange): CookieOp[] {\n switch (change.tag) {\n case 'update-table-metadata':\n return [\n {op: 'upsert-metadata', table: change.table, metadata: change.new},\n ];\n\n case 'create-table': {\n const {spec, metadata, backfill} = change;\n const table = {schema: spec.schema, name: spec.name};\n const ops: CookieOp[] = [];\n if (metadata) {\n ops.push({op: 'upsert-metadata', table, metadata});\n }\n if (backfill) {\n for (const [column, id] of Object.entries(backfill)) {\n ops.push({op: 'upsert-backfill', table, column, backfill: id});\n }\n }\n return ops;\n }\n\n case 'rename-table':\n return [{op: 'rename-table', old: change.old, new: change.new}];\n\n case 'drop-table':\n return [{op: 'drop-table', table: change.id}];\n\n case 'add-column': {\n const {table, tableMetadata, column, backfill} = change;\n const ops: CookieOp[] = [];\n if (tableMetadata) {\n ops.push({op: 'upsert-metadata', table, metadata: tableMetadata});\n }\n if (backfill) {\n ops.push({\n op: 'upsert-backfill',\n table,\n column: column.name,\n backfill,\n });\n }\n return ops;\n }\n\n case 'update-column': {\n // Only a rename moves the cookie; the rest of a column update does not\n // affect whether or how it is being backfilled.\n const {table, old, new: updated} = change;\n return old.name === updated.name\n ? []\n : [{op: 'rename-column', table, old: old.name, new: updated.name}];\n }\n\n case 'drop-column':\n return [{op: 'drop-column', table: change.table, column: change.column}];\n\n case 'backfill-completed': {\n const {relation, columns} = change;\n // The rowKey columns are excluded from `columns` but are backfilled with\n // them, so both are cleared.\n const cleared = [...relation.rowKey.columns, ...columns];\n return cleared.length === 0\n ? []\n : [\n {\n op: 'complete-backfill',\n table: {schema: relation.schema, name: relation.name},\n columns: cleared,\n },\n ];\n }\n\n case 'create-index':\n case 'drop-index':\n return [];\n\n default:\n // Unreachable: `isSchemaChange()` gates on the same tag set this switch\n // covers. It is exhaustive so that a new schema-change tag is a compile\n // error here rather than a silently dropped backfill.\n unreachable(change);\n }\n}\n\n/\n The SQLite interpreter of {@link cookieOps}, over prepared statements on the\n * change-log database.\n \n The statements run inside the log's transaction, in stream order, at the point\n * the schema change is appended (see `ChangeLogStreamWriter`). That is what\n * makes the cookie set atomic with the head — a crash cannot leave cookies ahead\n * of or behind the rows they were folded from — and what makes an aborted\n * transaction discard its cookies with its rows, for free.\n \n Statements are prepared against the tables as they exist now, so an instance\n * does not survive a reconcile that drops and recreates them.\n /\nexport class ChangeLogCookieWriter {\n readonly #upsertMetadata: Statement;\n readonly #upsertBackfill: Statement;\n readonly #renameMetadataTable: Statement;\n readonly #renameBackfillTable: Statement;\n readonly #deleteMetadataTable: Statement;\n readonly #deleteBackfillTable: Statement;\n readonly #renameBackfillColumn: Statement;\n readonly #deleteBackfillColumn: Statement;\n\n constructor(db: Database) {\n this.#upsertMetadata = db.prepare(/sql/ `\n INSERT INTO \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n (\"schema\", \"table\", \"metadata\") VALUES (?, ?, ?)\n ON CONFLICT (\"schema\", \"table\")\n DO UPDATE SET \"metadata\" = excluded.\"metadata\"\n `);\n this.#upsertBackfill = db.prepare(/sql/ `\n INSERT INTO \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n ON CONFLICT (\"schema\", \"table\", \"column\")\n DO UPDATE SET \"backfill\" = excluded.\"backfill\"\n `);\n this.#renameMetadataTable = db.prepare(/sql/ `\n UPDATE \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n `);\n this.#renameBackfillTable = db.prepare(/sql/ `\n UPDATE \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n `);\n this.#deleteMetadataTable = db.prepare(/sql/ `\n DELETE FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n WHERE \"schema\" = ? AND \"table\" = ?\n `);\n this.#deleteBackfillTable = db.prepare(/sql/ `\n DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n WHERE \"schema\" = ? AND \"table\" = ?\n `);\n this.#renameBackfillColumn = db.prepare(/sql/ `\n UPDATE \"${CHANGE_LOG_BACKFILLING_TABLE}\" SET \"column\" = ?\n WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n `);\n this.#deleteBackfillColumn = db.prepare(/sql/ `\n DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n `);\n }\n\n /* Applies the change's cookie ops, returning the ops that were applied. /\n apply(change: SchemaChange): CookieOp[] {\n const ops = cookieOps(change);\n for (const op of ops) {\n this.#run(op);\n }\n return ops;\n }\n\n #run(op: CookieOp): void {\n switch (op.op) {\n case 'upsert-metadata':\n this.#upsertMetadata.run(\n op.table.schema,\n op.table.name,\n BigIntJSON.stringify(op.metadata),\n );\n break;\n\n case 'upsert-backfill':\n this.#upsertBackfill.run(\n op.table.schema,\n op.table.name,\n op.column,\n BigIntJSON.stringify(op.backfill),\n );\n break;\n\n case 'rename-table':\n this.#renameMetadataTable.run(\n op.new.schema,\n op.new.name,\n op.old.schema,\n op.old.name,\n );\n this.#renameBackfillTable.run(\n op.new.schema,\n op.new.name,\n op.old.schema,\n op.old.name,\n );\n break;\n\n case 'drop-table':\n this.#deleteMetadataTable.run(op.table.schema, op.table.name);\n this.#deleteBackfillTable.run(op.table.schema, op.table.name);\n break;\n\n case 'rename-column':\n this.#renameBackfillColumn.run(\n op.new,\n op.table.schema,\n op.table.name,\n op.old,\n );\n break;\n\n case 'drop-column':\n this.#deleteBackfillColumn.run(\n op.table.schema,\n op.table.name,\n op.column,\n );\n break;\n\n case 'complete-backfill':\n // A per-column delete rather than an `IN` list, which cannot be a\n // single prepared statement across arities. The columns of one\n // completed backfill are few, and this runs only on schema changes.\n for (const column of op.columns) {\n this.#deleteBackfillColumn.run(\n op.table.schema,\n op.table.name,\n column,\n );\n }\n break;\n\n default:\n unreachable(op);\n }\n }\n}\n\nconst tableKey = (schema: string, table: string) =>\n JSON.stringify([schema, table]);\n\nconst columnKey = (schema: string, table: string, column: string) =>\n JSON.stringify([schema, table, column]);\n\nconst cmp = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0);\n\n/\n The in-memory interpreter of {@link cookieOps}: the third one, and the only\n * one that is not a store.\n \n A replica-derived cookie set is a snapshot at the replica's state version,\n * which trails the change log's head, so it cannot be compared with a\n * Postgres-derived set directly. This carries it forward over the schema changes\n * the log holds in between, which turns \"the two stores disagree, but they are\n * also at different positions\" into an exact equality — and makes a\n * disagreement between any two of the three interpreters visible as an\n * inequality that neither store would have noticed on its own.\n \n `changes` must be the log's schema changes over `(from, to]` in stream order.\n * Anything else produces a set that belongs to no watermark.\n /\nexport function foldCookies(\n cookies: CookieSet,\n changes: Iterable<SchemaChange>,\n): CookieSet {\n const metadata = new Map<string, TableMetadataCookie>(\n cookies.tableMetadata.map(c => [tableKey(c.schema, c.table), c]),\n );\n const backfilling = new Map<string, BackfillCookie>(\n cookies.backfilling.map(c => [columnKey(c.schema, c.table, c.column), c]),\n );\n const dropColumn = ({schema, name}: Identifier, column: string) =>\n backfilling.delete(columnKey(schema, name, column));\n\n for (const change of changes) {\n for (const op of cookieOps(change)) {\n switch (op.op) {\n case 'upsert-metadata':\n metadata.set(tableKey(op.table.schema, op.table.name), {\n schema: op.table.schema,\n table: op.table.name,\n metadata: op.metadata,\n });\n break;\n\n case 'upsert-backfill':\n backfilling.set(\n columnKey(op.table.schema, op.table.name, op.column),\n {\n schema: op.table.schema,\n table: op.table.name,\n column: op.column,\n backfill: op.backfill,\n },\n );\n break;\n\n case 'rename-table': {\n const {old, new: renamed} = op;\n const moved = metadata.get(tableKey(old.schema, old.name));\n if (moved) {\n metadata.delete(tableKey(old.schema, old.name));\n metadata.set(tableKey(renamed.schema, renamed.name), {\n ...moved,\n schema: renamed.schema,\n table: renamed.name,\n });\n }\n // Snapshotted before mutating, since the rename both deletes and\n // inserts into the map being scanned.\n for (const [key, cookie] of [...backfilling]) {\n if (cookie.schema === old.schema && cookie.table === old.name) {\n backfilling.delete(key);\n backfilling.set(\n columnKey(renamed.schema, renamed.name, cookie.column),\n {...cookie, schema: renamed.schema, table: renamed.name},\n );\n }\n }\n break;\n }\n\n case 'drop-table': {\n const {schema, name} = op.table;\n metadata.delete(tableKey(schema, name));\n for (const [key, cookie] of [...backfilling]) {\n if (cookie.schema === schema && cookie.table === name) {\n backfilling.delete(key);\n }\n }\n break;\n }\n\n case 'rename-column': {\n const {schema, name} = op.table;\n const moved = backfilling.get(columnKey(schema, name, op.old));\n if (moved) {\n backfilling.delete(columnKey(schema, name, op.old));\n backfilling.set(columnKey(schema, name, op.new), {\n ...moved,\n column: op.new,\n });\n }\n break;\n }\n\n case 'drop-column':\n dropColumn(op.table, op.column);\n break;\n\n case 'complete-backfill':\n for (const column of op.columns) {\n dropColumn(op.table, column);\n }\n break;\n\n default:\n unreachable(op);\n }\n }\n }\n\n return {\n tableMetadata: [...metadata.values()].toSorted(\n (a, b) => cmp(a.schema, b.schema) \|\| cmp(a.table, b.table),\n ),\n backfilling: [...backfilling.values()].toSorted(\n (a, b) =>\n cmp(a.schema, b.schema) \|\|\n cmp(a.table, b.table) \|\|\n cmp(a.column, b.column),\n ),\n };\n}\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\n/\n Returns one {@link BackfillRequest} for each table with an active backfill.\n * Each request contains the stored table metadata, if available.\n /\nexport function backfillRequestsFrom(cookies: CookieSet): BackfillRequest[] {\n const metadata = new Map(\n cookies.tableMetadata.map(c => [tableKey(c.schema, c.table), c.metadata]),\n );\n const requests: BackfillRequest[] = [];\n let curr: BackfillRequest \| undefined;\n for (const {schema, table, column, backfill} of cookies.backfilling) {\n if (curr?.table.schema !== schema \|\| curr.table.name !== table) {\n curr = {\n table: {\n schema,\n name: table,\n // Use `null` when a backfilling table has no metadata. This matches\n // the result of the Postgres left join.\n metadata: metadata.get(tableKey(schema, table)) ?? null,\n },\n columns: {},\n };\n requests.push(curr);\n }\n curr.columns[column] = backfill;\n }\n return v.parse(requests, backfillRequestsSchema);\n}\n\n/\n Reads the whole cookie set, ordered by primary key. This is what a\n * change-streamer would hand `startStream`, and what the Postgres-derived and\n * replica-derived sets are compared against.\n /\nexport function readCookies(db: Database): CookieSet {\n const tableMetadata = db\n .prepare(/sql/ `\n SELECT \"schema\", \"table\", \"metadata\"\n FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n ORDER BY \"schema\", \"table\"\n `)\n .all<{schema: string; table: string; metadata: string}>()\n .map(({schema, table, metadata}) => ({\n schema,\n table,\n metadata: BigIntJSON.parse(metadata) as TableMetadata,\n }));\n\n const backfilling = db\n .prepare(/sql/ `\n SELECT \"schema\", \"table\", \"column\", \"backfill\"\n FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n ORDER BY \"schema\", \"table\", \"column\"\n `)\n .all<{schema: string; table: string; column: string; backfill: string}>()\n .map(({schema, table, column, backfill}) => ({\n schema,\n table,\n column,\n backfill: BigIntJSON.parse(backfill) as BackfillID,\n }));\n\n return {tableMetadata, backfilling};\n}\n\n/\n Replaces the cookie set wholesale.\n \n The cookies are unversioned point-in-time state, so a reconciliation that\n * moves the head invalidates them: truncating the stream above `W` does not roll\n * the cookie set back to `W`, and the truncated transactions can contain\n * `create-table`, `add-column`, and `backfill-completed`. The caller must run\n * this in the same transaction as the reconciliation that invalidated them.\n /\nexport function replaceCookies(db: Database, cookies: CookieSet): void {\n db.prepare(/sql/ `DELETE FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"`).run();\n db.prepare(/sql/ `DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"`).run();\n\n if (cookies.tableMetadata.length > 0) {\n const insert = db.prepare(/sql/ `\n INSERT INTO \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n (\"schema\", \"table\", \"metadata\") VALUES (?, ?, ?)\n `);\n for (const {schema, table, metadata} of cookies.tableMetadata) {\n insert.run(schema, table, BigIntJSON.stringify(metadata));\n }\n }\n if (cookies.backfilling.length > 0) {\n const insert = db.prepare(/sql/ `\n INSERT INTO \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n `);\n for (const {schema, table, column, backfill} of cookies.backfilling) {\n insert.run(schema, table, column, BigIntJSON.stringify(backfill));\n }\n }\n}\n\n/* Retained cookie rows, by table. Both are normally zero. /\nexport type CookieRowCounts = {\n readonly tableMetadata: number;\n readonly backfilling: number;\n};\n\nexport function readCookieRowCounts(db: Database): CookieRowCounts {\n return db\n .prepare(/sql/ `\n SELECT\n (SELECT count() FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\")\n AS \"tableMetadata\",\n (SELECT count(*) FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\")\n AS \"backfilling\"\n `)\n .get<CookieRowCounts>();\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,IAAa,kCAAkC;AAC/C,IAAa,+BAA+B;AAQ5C,IAAa,kCAA0C;kBACrC,gCAAgC;;;;;;;kBAOhC,6BAA6B;;;;;;;;AAS/C,IAAa,gCAAwC;0BAC3B,gCAAgC;0BAChC,6BAA6B;;;;;;;;;;;AA4DvD,SAAgB,UAAU,QAAkC;CAC1D,QAAQ,OAAO,KAAf;EACE,KAAK,yBACH,OAAO,CACL;GAAC,IAAI;GAAmB,OAAO,OAAO;GAAO,UAAU,OAAO;EAAG,CACnE;EAEF,KAAK,gBAAgB;GACnB,MAAM,EAAC,MAAM,UAAU,aAAY;GACnC,MAAM,QAAQ;IAAC,QAAQ,KAAK;IAAQ,MAAM,KAAK;GAAI;GACnD,MAAM,MAAkB,CAAC;GACzB,IAAI,UACF,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO;GAAQ,CAAC;GAEnD,IAAI,UACF,KAAK,MAAM,CAAC,QAAQ,OAAO,OAAO,QAAQ,QAAQ,GAChD,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO;IAAQ,UAAU;GAAE,CAAC;GAGjE,OAAO;EACT;EAEA,KAAK,gBACH,OAAO,CAAC;GAAC,IAAI;GAAgB,KAAK,OAAO;GAAK,KAAK,OAAO;EAAG,CAAC;EAEhE,KAAK,cACH,OAAO,CAAC;GAAC,IAAI;GAAc,OAAO,OAAO;EAAE,CAAC;EAE9C,KAAK,cAAc;GACjB,MAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;GACjD,MAAM,MAAkB,CAAC;GACzB,IAAI,eACF,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO,UAAU;GAAa,CAAC;GAElE,IAAI,UACF,IAAI,KAAK;IACP,IAAI;IACJ;IACA,QAAQ,OAAO;IACf;GACF,CAAC;GAEH,OAAO;EACT;EAEA,KAAK,iBAAiB;GAGpB,MAAM,EAAC,OAAO,KAAK,KAAK,YAAW;GACnC,OAAO,IAAI,SAAS,QAAQ,OACxB,CAAC,IACD,CAAC;IAAC,IAAI;IAAiB;IAAO,KAAK,IAAI;IAAM,KAAK,QAAQ;GAAI,CAAC;EACrE;EAEA,KAAK,eACH,OAAO,CAAC;GAAC,IAAI;GAAe,OAAO,OAAO;GAAO,QAAQ,OAAO;EAAM,CAAC;EAEzE,KAAK,sBAAsB;GACzB,MAAM,EAAC,UAAU,YAAW;GAG5B,MAAM,UAAU,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;GACvD,OAAO,QAAQ,WAAW,IACtB,CAAC,IACD,CACE;IACE,IAAI;IACJ,OAAO;KAAC,QAAQ,SAAS;KAAQ,MAAM,SAAS;IAAI;IACpD,SAAS;GACX,CACF;EACN;EAEA,KAAK;EACL,KAAK,cACH,OAAO,CAAC;EAEV,SAIE,YAAY,MAAM;CACtB;AACF;;;;;;;;;;;;;;AAeA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAKA,kBAAkB,GAAG,QAAgB;qBACzB,gCAAgC;;;;KAIhD;EACD,KAAKC,kBAAkB,GAAG,QAAgB;qBACzB,6BAA6B;;;;KAI7C;EACD,KAAKC,uBAAuB,GAAG,QAAgB;gBACnC,gCAAgC;;KAE3C;EACD,KAAKC,uBAAuB,GAAG,QAAgB;gBACnC,6BAA6B;;KAExC;EACD,KAAKC,uBAAuB,GAAG,QAAgB;qBAC9B,gCAAgC;;KAEhD;EACD,KAAKC,uBAAuB,GAAG,QAAgB;qBAC9B,6BAA6B;;KAE7C;EACD,KAAKC,wBAAwB,GAAG,QAAgB;gBACpC,6BAA6B;;KAExC;EACD,KAAKC,wBAAwB,GAAG,QAAgB;qBAC/B,6BAA6B;;KAE7C;CACH;;CAGA,MAAM,QAAkC;EACtC,MAAM,MAAM,UAAU,MAAM;EAC5B,KAAK,MAAM,MAAM,KACf,KAAKC,KAAK,EAAE;EAEd,OAAO;CACT;CAEA,KAAK,IAAoB;EACvB,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,KAAKR,gBAAgB,IACnB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,KAAKC,gBAAgB,IACnB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,QACH,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,KAAKC,qBAAqB,IACxB,GAAG,IAAI,QACP,GAAG,IAAI,MACP,GAAG,IAAI,QACP,GAAG,IAAI,IACT;IACA,KAAKC,qBAAqB,IACxB,GAAG,IAAI,QACP,GAAG,IAAI,MACP,GAAG,IAAI,QACP,GAAG,IAAI,IACT;IACA;GAEF,KAAK;IACH,KAAKC,qBAAqB,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IAC5D,KAAKC,qBAAqB,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IAC5D;GAEF,KAAK;IACH,KAAKC,sBAAsB,IACzB,GAAG,KACH,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,GACL;IACA;GAEF,KAAK;IACH,KAAKC,sBAAsB,IACzB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,MACL;IACA;GAEF,KAAK;IAIH,KAAK,MAAM,UAAU,GAAG,SACtB,KAAKA,sBAAsB,IACzB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,MACF;IAEF;GAEF,SACE,YAAY,EAAE;EAClB;CACF;AACF;AAEA,IAAM,YAAY,QAAgB,UAChC,KAAK,UAAU,CAAC,QAAQ,KAAK,CAAC;AAEhC,IAAM,aAAa,QAAgB,OAAe,WAChD,KAAK,UAAU;CAAC;CAAQ;CAAO;AAAM,CAAC;AAExC,IAAM,OAAO,GAAW,MAAe,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI;;;;;;;;;;;;;;;;AAiBhE,SAAgB,YACd,SACA,SACW;CACX,MAAM,WAAW,IAAI,IACnB,QAAQ,cAAc,KAAI,MAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CACjE;CACA,MAAM,cAAc,IAAI,IACtB,QAAQ,YAAY,KAAI,MAAK,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,CAC1E;CACA,MAAM,cAAc,EAAC,QAAQ,QAAmB,WAC9C,YAAY,OAAO,UAAU,QAAQ,MAAM,MAAM,CAAC;CAEpD,KAAK,MAAM,UAAU,SACnB,KAAK,MAAM,MAAM,UAAU,MAAM,GAC/B,QAAQ,GAAG,IAAX;EACE,KAAK;GACH,SAAS,IAAI,SAAS,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI,GAAG;IACrD,QAAQ,GAAG,MAAM;IACjB,OAAO,GAAG,MAAM;IAChB,UAAU,GAAG;GACf,CAAC;GACD;EAEF,KAAK;GACH,YAAY,IACV,UAAU,GAAG,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,MAAM,GACnD;IACE,QAAQ,GAAG,MAAM;IACjB,OAAO,GAAG,MAAM;IAChB,QAAQ,GAAG;IACX,UAAU,GAAG;GACf,CACF;GACA;EAEF,KAAK,gBAAgB;GACnB,MAAM,EAAC,KAAK,KAAK,YAAW;GAC5B,MAAM,QAAQ,SAAS,IAAI,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC;GACzD,IAAI,OAAO;IACT,SAAS,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC;IAC9C,SAAS,IAAI,SAAS,QAAQ,QAAQ,QAAQ,IAAI,GAAG;KACnD,GAAG;KACH,QAAQ,QAAQ;KAChB,OAAO,QAAQ;IACjB,CAAC;GACH;GAGA,KAAK,MAAM,CAAC,KAAK,WAAW,CAAC,GAAG,WAAW,GACzC,IAAI,OAAO,WAAW,IAAI,UAAU,OAAO,UAAU,IAAI,MAAM;IAC7D,YAAY,OAAO,GAAG;IACtB,YAAY,IACV,UAAU,QAAQ,QAAQ,QAAQ,MAAM,OAAO,MAAM,GACrD;KAAC,GAAG;KAAQ,QAAQ,QAAQ;KAAQ,OAAO,QAAQ;IAAI,CACzD;GACF;GAEF;EACF;EAEA,KAAK,cAAc;GACjB,MAAM,EAAC,QAAQ,SAAQ,GAAG;GAC1B,SAAS,OAAO,SAAS,QAAQ,IAAI,CAAC;GACtC,KAAK,MAAM,CAAC,KAAK,WAAW,CAAC,GAAG,WAAW,GACzC,IAAI,OAAO,WAAW,UAAU,OAAO,UAAU,MAC/C,YAAY,OAAO,GAAG;GAG1B;EACF;EAEA,KAAK,iBAAiB;GACpB,MAAM,EAAC,QAAQ,SAAQ,GAAG;GAC1B,MAAM,QAAQ,YAAY,IAAI,UAAU,QAAQ,MAAM,GAAG,GAAG,CAAC;GAC7D,IAAI,OAAO;IACT,YAAY,OAAO,UAAU,QAAQ,MAAM,GAAG,GAAG,CAAC;IAClD,YAAY,IAAI,UAAU,QAAQ,MAAM,GAAG,GAAG,GAAG;KAC/C,GAAG;KACH,QAAQ,GAAG;IACb,CAAC;GACH;GACA;EACF;EAEA,KAAK;GACH,WAAW,GAAG,OAAO,GAAG,MAAM;GAC9B;EAEF,KAAK;GACH,KAAK,MAAM,UAAU,GAAG,SACtB,WAAW,GAAG,OAAO,MAAM;GAE7B;EAEF,SACE,YAAY,EAAE;CAClB;CAIJ,OAAO;EACL,eAAe,CAAC,GAAG,SAAS,OAAO,CAAC,EAAE,UACnC,GAAG,MAAM,IAAI,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,EAAE,OAAO,EAAE,KAAK,CAC3D;EACA,aAAa,CAAC,GAAG,YAAY,OAAO,CAAC,EAAE,UACpC,GAAG,MACF,IAAI,EAAE,QAAQ,EAAE,MAAM,KACtB,IAAI,EAAE,OAAO,EAAE,KAAK,KACpB,IAAI,EAAE,QAAQ,EAAE,MAAM,CAC1B;CACF;AACF;AAEA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;AAM5D,SAAgB,qBAAqB,SAAuC;CAC1E,MAAM,WAAW,IAAI,IACnB,QAAQ,cAAc,KAAI,MAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,GAAG,EAAE,QAAQ,CAAC,CAC1E;CACA,MAAM,WAA8B,CAAC;CACrC,IAAI;CACJ,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,cAAa,QAAQ,aAAa;EACnE,IAAI,MAAM,MAAM,WAAW,UAAU,KAAK,MAAM,SAAS,OAAO;GAC9D,OAAO;IACL,OAAO;KACL;KACA,MAAM;KAGN,UAAU,SAAS,IAAI,SAAS,QAAQ,KAAK,CAAC,KAAK;IACrD;IACA,SAAS,CAAC;GACZ;GACA,SAAS,KAAK,IAAI;EACpB;EACA,KAAK,QAAQ,UAAU;CACzB;CACA,OAAO,MAAQ,UAAU,sBAAsB;AACjD;;;;;;AAOA,SAAgB,YAAY,IAAyB;CA4BnD,OAAO;EAAC,eA3Bc,GACnB,QAAgB;;gBAEL,gCAAgC;;KAE3C,EACA,IAAuD,EACvD,KAAK,EAAC,QAAQ,OAAO,gBAAe;GACnC;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAgBM;EAAe,aAdH,GACjB,QAAgB;;gBAEL,6BAA6B;;KAExC,EACA,IAAuE,EACvE,KAAK,EAAC,QAAQ,OAAO,QAAQ,gBAAe;GAC3C;GACA;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAEqB;CAAW;AACpC;;;;;;;;;;AAWA,SAAgB,eAAe,IAAc,SAA0B;CACrE,GAAG,QAAgB,gBAAgB,gCAAgC,EAAE,EAAE,IAAI;CAC3E,GAAG,QAAgB,gBAAgB,6BAA6B,EAAE,EAAE,IAAI;CAExE,IAAI,QAAQ,cAAc,SAAS,GAAG;EACpC,MAAM,SAAS,GAAG,QAAgB;qBACjB,gCAAgC;;KAEhD;EACD,KAAK,MAAM,EAAC,QAAQ,OAAO,cAAa,QAAQ,eAC9C,OAAO,IAAI,QAAQ,OAAO,WAAW,UAAU,QAAQ,CAAC;CAE5D;CACA,IAAI,QAAQ,YAAY,SAAS,GAAG;EAClC,MAAM,SAAS,GAAG,QAAgB;qBACjB,6BAA6B;;KAE7C;EACD,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,cAAa,QAAQ,aACtD,OAAO,IAAI,QAAQ,OAAO,QAAQ,WAAW,UAAU,QAAQ,CAAC;CAEpE;AACF;AAQA,SAAgB,oBAAoB,IAA+B;CACjE,OAAO,GACJ,QAAgB;;iCAEY,gCAAgC;;iCAEhC,6BAA6B;;KAEzD,EACA,IAAqB;AAC1B"}
1	+ {"version":3,"file":"change-log-cookies.js","names":["#upsertMetadata","#upsertBackfill","#renameMetadataTable","#renameBackfillTable","#deleteMetadataTable","#deleteBackfillTable","#renameBackfillColumn","#deleteBackfillColumn","#run"],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-cookies.ts"],"sourcesContent":["/*\n The change log's second job.\n \n A change log is a buffer and a cookie jar. The buffer catches a\n * reconnecting subscriber up to the head; the cookie jar holds the backfill\n * progress state that the change-streamer hands the change-source on every\n * `startStream`, so that the change source never has to persist backfill state\n * upstream. See `change-log-db.ts` for the buffer.\n \n There are two cookies, mirroring the two `cdc` tables the Postgres change log\n * has always carried (`change-streamer/schema/tables.ts`):\n \n - table metadata, the most current {@link TableMetadata} of a table, which\n * a {@link BackfillRequest} has to carry; and\n * - backfilling columns, the opaque {@link BackfillID} of every column whose\n * backfill has not completed.\n \n Both are a fold over the change stream: the cookie set is only meaningful\n * paired with the watermark it was folded up to. {@link cookieOps} is that fold,\n * expressed transport-free so that the Postgres change log and the SQLite change\n * log cannot drift from each other — they must agree on all eight transitions\n * forever, and the failure mode if they disagree is a backfill that is silently\n * never re-requested. The two interpreters are {@link ChangeLogCookieWriter}\n * here and `Storer.#trackBackfillMetadata()` in\n * `change-streamer/storer.ts`.\n \n The tables live in the change-log database rather than in the replica: the\n * replicator is a subscriber downstream of the change-streamer's write loop,\n * so its state version trails the log's head, and pairing its cookies with the\n * log's resume watermark would lose every backfill that completed in between.\n * The replica's own copies (`_zero.tableMetadata`, `_zero.column_metadata`) are\n * the seed source; these are the working set.\n /\n\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport as v from '../../../../shared/src/valita.ts';\nimport type {Database, Statement} from '../../../../zqlite/src/db.ts';\nimport type {\n BackfillID,\n Identifier,\n SchemaChange,\n TableMetadata,\n} from '../change-source/protocol/current/data.ts';\nimport {\n backfillRequestSchema,\n type BackfillRequest,\n} from '../change-source/protocol/current/upstream.ts';\n\nexport const CHANGE_LOG_TABLE_METADATA_TABLE = '_zero.changeLogTableMetadata';\nexport const CHANGE_LOG_BACKFILLING_TABLE = '_zero.changeLogBackfilling';\n\n// The names follow the file's existing convention, beside\n// \"_zero.changeLogStream\" and \"_zero.changeLogMeta\".\n//\n// `metadata` and `backfill` hold the JSON that the `cdc` tables hold as JSONB.\n// SQLite has no JSONB, and nothing here queries into the documents: they are\n// stored to be handed back to the change source verbatim.\nexport const CREATE_CHANGE_LOG_COOKIE_SCHEMA = /sql/ `\n CREATE TABLE \"${CHANGE_LOG_TABLE_METADATA_TABLE}\" (\n \"schema\" TEXT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"metadata\" TEXT NOT NULL,\n PRIMARY KEY (\"schema\", \"table\")\n );\n\n CREATE TABLE \"${CHANGE_LOG_BACKFILLING_TABLE}\" (\n \"schema\" TEXT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"column\" TEXT NOT NULL,\n \"backfill\" TEXT NOT NULL,\n PRIMARY KEY (\"schema\", \"table\", \"column\")\n );\n`;\n\nexport const DROP_CHANGE_LOG_COOKIE_TABLES = /sql/ `\n DROP TABLE IF EXISTS \"${CHANGE_LOG_TABLE_METADATA_TABLE}\";\n DROP TABLE IF EXISTS \"${CHANGE_LOG_BACKFILLING_TABLE}\";\n`;\n\nexport type TableMetadataCookie = {\n readonly schema: string;\n readonly table: string;\n readonly metadata: TableMetadata;\n};\n\nexport type BackfillCookie = {\n readonly schema: string;\n readonly table: string;\n readonly column: string;\n readonly backfill: BackfillID;\n};\n\n/*\n The whole cookie set, as of the log's head. Both lists are ordered by their\n * primary key so that two stores' sets can be compared directly.\n /\nexport type CookieSet = {\n readonly tableMetadata: readonly TableMetadataCookie[];\n readonly backfilling: readonly BackfillCookie[];\n};\n\nexport const EMPTY_COOKIE_SET: CookieSet = {\n tableMetadata: [],\n backfilling: [],\n};\n\n/\n A transport-free description of a cookie mutation. This is the whole of what\n * a {@link SchemaChange} means to the cookie jar; everything else about the\n * change belongs to the buffer.\n /\nexport type CookieOp =\n \| {op: 'upsert-metadata'; table: Identifier; metadata: TableMetadata}\n \| {\n op: 'upsert-backfill';\n table: Identifier;\n column: string;\n backfill: BackfillID;\n }\n \| {op: 'rename-table'; old: Identifier; new: Identifier}\n \| {op: 'drop-table'; table: Identifier}\n \| {op: 'rename-column'; table: Identifier; old: string; new: string}\n \| {op: 'drop-column'; table: Identifier; column: string}\n \| {op: 'complete-backfill'; table: Identifier; columns: readonly string[]};\n\nexport type CookieOpTag = CookieOp['op'];\n\n/\n The fold, and the only place either store decides what a schema change means\n * for the cookie set. Both interpreters are mechanical translations of what this\n * returns, in the order it returns them.\n \n Changes that carry no cookie state — index changes, and the `create-table` /\n * `add-column` variants whose optional `metadata` and `backfill` fields are\n * absent — produce no ops.\n /\nexport function cookieOps(change: SchemaChange): CookieOp[] {\n switch (change.tag) {\n case 'update-table-metadata':\n return [\n {op: 'upsert-metadata', table: change.table, metadata: change.new},\n ];\n\n case 'create-table': {\n const {spec, metadata, backfill} = change;\n const table = {schema: spec.schema, name: spec.name};\n const ops: CookieOp[] = [];\n if (metadata) {\n ops.push({op: 'upsert-metadata', table, metadata});\n }\n if (backfill) {\n for (const [column, id] of Object.entries(backfill)) {\n ops.push({op: 'upsert-backfill', table, column, backfill: id});\n }\n }\n return ops;\n }\n\n case 'rename-table':\n return [{op: 'rename-table', old: change.old, new: change.new}];\n\n case 'drop-table':\n return [{op: 'drop-table', table: change.id}];\n\n case 'add-column': {\n const {table, tableMetadata, column, backfill} = change;\n const ops: CookieOp[] = [];\n if (tableMetadata) {\n ops.push({op: 'upsert-metadata', table, metadata: tableMetadata});\n }\n if (backfill) {\n ops.push({\n op: 'upsert-backfill',\n table,\n column: column.name,\n backfill,\n });\n }\n return ops;\n }\n\n case 'update-column': {\n // Only a rename moves the cookie; the rest of a column update does not\n // affect whether or how it is being backfilled.\n const {table, old, new: updated} = change;\n return old.name === updated.name\n ? []\n : [{op: 'rename-column', table, old: old.name, new: updated.name}];\n }\n\n case 'drop-column':\n return [{op: 'drop-column', table: change.table, column: change.column}];\n\n case 'backfill-completed': {\n const {relation, columns} = change;\n // The rowKey columns are excluded from `columns` but are backfilled with\n // them, so both are cleared.\n const cleared = [...relation.rowKey.columns, ...columns];\n return cleared.length === 0\n ? []\n : [\n {\n op: 'complete-backfill',\n table: {schema: relation.schema, name: relation.name},\n columns: cleared,\n },\n ];\n }\n\n case 'create-index':\n case 'drop-index':\n return [];\n\n default:\n // Unreachable: `isSchemaChange()` gates on the same tag set this switch\n // covers. It is exhaustive so that a new schema-change tag is a compile\n // error here rather than a silently dropped backfill.\n unreachable(change);\n }\n}\n\n/\n The SQLite interpreter of {@link cookieOps}, over prepared statements on the\n * change-log database.\n \n The statements run inside the log's transaction, in stream order, at the point\n * the schema change is appended (see `ChangeLogStreamWriter`). That is what\n * makes the cookie set atomic with the head — a crash cannot leave cookies ahead\n * of or behind the rows they were folded from — and what makes an aborted\n * transaction discard its cookies with its rows, for free.\n \n Statements are prepared against the tables as they exist now, so an instance\n * does not survive a reconcile that drops and recreates them.\n /\nexport class ChangeLogCookieWriter {\n readonly #upsertMetadata: Statement;\n readonly #upsertBackfill: Statement;\n readonly #renameMetadataTable: Statement;\n readonly #renameBackfillTable: Statement;\n readonly #deleteMetadataTable: Statement;\n readonly #deleteBackfillTable: Statement;\n readonly #renameBackfillColumn: Statement;\n readonly #deleteBackfillColumn: Statement;\n\n constructor(db: Database) {\n this.#upsertMetadata = db.prepare(/sql/ `\n INSERT INTO \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n (\"schema\", \"table\", \"metadata\") VALUES (?, ?, ?)\n ON CONFLICT (\"schema\", \"table\")\n DO UPDATE SET \"metadata\" = excluded.\"metadata\"\n `);\n this.#upsertBackfill = db.prepare(/sql/ `\n INSERT INTO \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n ON CONFLICT (\"schema\", \"table\", \"column\")\n DO UPDATE SET \"backfill\" = excluded.\"backfill\"\n `);\n this.#renameMetadataTable = db.prepare(/sql/ `\n UPDATE \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n `);\n this.#renameBackfillTable = db.prepare(/sql/ `\n UPDATE \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n `);\n this.#deleteMetadataTable = db.prepare(/sql/ `\n DELETE FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n WHERE \"schema\" = ? AND \"table\" = ?\n `);\n this.#deleteBackfillTable = db.prepare(/sql/ `\n DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n WHERE \"schema\" = ? AND \"table\" = ?\n `);\n this.#renameBackfillColumn = db.prepare(/sql/ `\n UPDATE \"${CHANGE_LOG_BACKFILLING_TABLE}\" SET \"column\" = ?\n WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n `);\n this.#deleteBackfillColumn = db.prepare(/sql/ `\n DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n `);\n }\n\n /* Applies the change's cookie ops, returning the ops that were applied. /\n apply(change: SchemaChange): CookieOp[] {\n const ops = cookieOps(change);\n for (const op of ops) {\n this.#run(op);\n }\n return ops;\n }\n\n #run(op: CookieOp): void {\n switch (op.op) {\n case 'upsert-metadata':\n this.#upsertMetadata.run(\n op.table.schema,\n op.table.name,\n BigIntJSON.stringify(op.metadata),\n );\n break;\n\n case 'upsert-backfill':\n this.#upsertBackfill.run(\n op.table.schema,\n op.table.name,\n op.column,\n BigIntJSON.stringify(op.backfill),\n );\n break;\n\n case 'rename-table':\n this.#renameMetadataTable.run(\n op.new.schema,\n op.new.name,\n op.old.schema,\n op.old.name,\n );\n this.#renameBackfillTable.run(\n op.new.schema,\n op.new.name,\n op.old.schema,\n op.old.name,\n );\n break;\n\n case 'drop-table':\n this.#deleteMetadataTable.run(op.table.schema, op.table.name);\n this.#deleteBackfillTable.run(op.table.schema, op.table.name);\n break;\n\n case 'rename-column':\n this.#renameBackfillColumn.run(\n op.new,\n op.table.schema,\n op.table.name,\n op.old,\n );\n break;\n\n case 'drop-column':\n this.#deleteBackfillColumn.run(\n op.table.schema,\n op.table.name,\n op.column,\n );\n break;\n\n case 'complete-backfill':\n // A per-column delete rather than an `IN` list, which cannot be a\n // single prepared statement across arities. The columns of one\n // completed backfill are few, and this runs only on schema changes.\n for (const column of op.columns) {\n this.#deleteBackfillColumn.run(\n op.table.schema,\n op.table.name,\n column,\n );\n }\n break;\n\n default:\n unreachable(op);\n }\n }\n}\n\nconst tableKey = (schema: string, table: string) =>\n JSON.stringify([schema, table]);\n\nconst columnKey = (schema: string, table: string, column: string) =>\n JSON.stringify([schema, table, column]);\n\nconst cmp = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0);\n\n/\n The in-memory interpreter of {@link cookieOps}: the third one, and the only\n * one that is not a store.\n \n A replica-derived cookie set is a snapshot at the replica's state version,\n * which trails the change log's head, so it cannot be compared with a\n * Postgres-derived set directly. This carries it forward over the schema changes\n * the log holds in between, which turns \"the two stores disagree, but they are\n * also at different positions\" into an exact equality — and makes a\n * disagreement between any two of the three interpreters visible as an\n * inequality that neither store would have noticed on its own.\n \n `changes` must be the log's schema changes over `(from, to]` in stream order.\n * Anything else produces a set that belongs to no watermark.\n /\nexport function foldCookies(\n cookies: CookieSet,\n changes: Iterable<SchemaChange>,\n): CookieSet {\n const metadata = new Map<string, TableMetadataCookie>(\n cookies.tableMetadata.map(c => [tableKey(c.schema, c.table), c]),\n );\n const backfilling = new Map<string, BackfillCookie>(\n cookies.backfilling.map(c => [columnKey(c.schema, c.table, c.column), c]),\n );\n const dropColumn = ({schema, name}: Identifier, column: string) =>\n backfilling.delete(columnKey(schema, name, column));\n\n for (const change of changes) {\n for (const op of cookieOps(change)) {\n switch (op.op) {\n case 'upsert-metadata':\n metadata.set(tableKey(op.table.schema, op.table.name), {\n schema: op.table.schema,\n table: op.table.name,\n metadata: op.metadata,\n });\n break;\n\n case 'upsert-backfill':\n backfilling.set(\n columnKey(op.table.schema, op.table.name, op.column),\n {\n schema: op.table.schema,\n table: op.table.name,\n column: op.column,\n backfill: op.backfill,\n },\n );\n break;\n\n case 'rename-table': {\n const {old, new: renamed} = op;\n const moved = metadata.get(tableKey(old.schema, old.name));\n if (moved) {\n metadata.delete(tableKey(old.schema, old.name));\n metadata.set(tableKey(renamed.schema, renamed.name), {\n ...moved,\n schema: renamed.schema,\n table: renamed.name,\n });\n }\n // Snapshotted before mutating, since the rename both deletes and\n // inserts into the map being scanned.\n for (const [key, cookie] of [...backfilling]) {\n if (cookie.schema === old.schema && cookie.table === old.name) {\n backfilling.delete(key);\n backfilling.set(\n columnKey(renamed.schema, renamed.name, cookie.column),\n {...cookie, schema: renamed.schema, table: renamed.name},\n );\n }\n }\n break;\n }\n\n case 'drop-table': {\n const {schema, name} = op.table;\n metadata.delete(tableKey(schema, name));\n for (const [key, cookie] of [...backfilling]) {\n if (cookie.schema === schema && cookie.table === name) {\n backfilling.delete(key);\n }\n }\n break;\n }\n\n case 'rename-column': {\n const {schema, name} = op.table;\n const moved = backfilling.get(columnKey(schema, name, op.old));\n if (moved) {\n backfilling.delete(columnKey(schema, name, op.old));\n backfilling.set(columnKey(schema, name, op.new), {\n ...moved,\n column: op.new,\n });\n }\n break;\n }\n\n case 'drop-column':\n dropColumn(op.table, op.column);\n break;\n\n case 'complete-backfill':\n for (const column of op.columns) {\n dropColumn(op.table, column);\n }\n break;\n\n default:\n unreachable(op);\n }\n }\n }\n\n return {\n tableMetadata: [...metadata.values()].toSorted(\n (a, b) => cmp(a.schema, b.schema) \|\| cmp(a.table, b.table),\n ),\n backfilling: [...backfilling.values()].toSorted(\n (a, b) =>\n cmp(a.schema, b.schema) \|\|\n cmp(a.table, b.table) \|\|\n cmp(a.column, b.column),\n ),\n };\n}\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\n/\n Returns one {@link BackfillRequest} for each table with an active backfill.\n * Each request contains the stored table metadata, if available.\n /\nexport function backfillRequestsFrom(cookies: CookieSet): BackfillRequest[] {\n const metadata = new Map(\n cookies.tableMetadata.map(c => [tableKey(c.schema, c.table), c.metadata]),\n );\n const requests: BackfillRequest[] = [];\n let curr: BackfillRequest \| undefined;\n for (const {schema, table, column, backfill} of cookies.backfilling) {\n if (curr?.table.schema !== schema \|\| curr.table.name !== table) {\n curr = {\n table: {\n schema,\n name: table,\n // Use `null` when a backfilling table has no metadata. This matches\n // the result of the Postgres left join.\n metadata: metadata.get(tableKey(schema, table)) ?? null,\n },\n columns: {},\n };\n requests.push(curr);\n }\n curr.columns[column] = {id: backfill};\n }\n return v.parse(requests, backfillRequestsSchema);\n}\n\n/\n Reads the whole cookie set, ordered by primary key. This is what a\n * change-streamer would hand `startStream`, and what the Postgres-derived and\n * replica-derived sets are compared against.\n /\nexport function readCookies(db: Database): CookieSet {\n const tableMetadata = db\n .prepare(/sql/ `\n SELECT \"schema\", \"table\", \"metadata\"\n FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n ORDER BY \"schema\", \"table\"\n `)\n .all<{schema: string; table: string; metadata: string}>()\n .map(({schema, table, metadata}) => ({\n schema,\n table,\n metadata: BigIntJSON.parse(metadata) as TableMetadata,\n }));\n\n const backfilling = db\n .prepare(/sql/ `\n SELECT \"schema\", \"table\", \"column\", \"backfill\"\n FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n ORDER BY \"schema\", \"table\", \"column\"\n `)\n .all<{schema: string; table: string; column: string; backfill: string}>()\n .map(({schema, table, column, backfill}) => ({\n schema,\n table,\n column,\n backfill: BigIntJSON.parse(backfill) as BackfillID,\n }));\n\n return {tableMetadata, backfilling};\n}\n\n/\n Replaces the cookie set wholesale.\n \n The cookies are unversioned point-in-time state, so a reconciliation that\n * moves the head invalidates them: truncating the stream above `W` does not roll\n * the cookie set back to `W`, and the truncated transactions can contain\n * `create-table`, `add-column`, and `backfill-completed`. The caller must run\n * this in the same transaction as the reconciliation that invalidated them.\n /\nexport function replaceCookies(db: Database, cookies: CookieSet): void {\n db.prepare(/sql/ `DELETE FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"`).run();\n db.prepare(/sql/ `DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"`).run();\n\n if (cookies.tableMetadata.length > 0) {\n const insert = db.prepare(/sql/ `\n INSERT INTO \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n (\"schema\", \"table\", \"metadata\") VALUES (?, ?, ?)\n `);\n for (const {schema, table, metadata} of cookies.tableMetadata) {\n insert.run(schema, table, BigIntJSON.stringify(metadata));\n }\n }\n if (cookies.backfilling.length > 0) {\n const insert = db.prepare(/sql/ `\n INSERT INTO \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n `);\n for (const {schema, table, column, backfill} of cookies.backfilling) {\n insert.run(schema, table, column, BigIntJSON.stringify(backfill));\n }\n }\n}\n\n/* Retained cookie rows, by table. Both are normally zero. /\nexport type CookieRowCounts = {\n readonly tableMetadata: number;\n readonly backfilling: number;\n};\n\nexport function readCookieRowCounts(db: Database): CookieRowCounts {\n return db\n .prepare(/sql/ `\n SELECT\n (SELECT count() FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\")\n AS \"tableMetadata\",\n (SELECT count(*) FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\")\n AS \"backfilling\"\n `)\n .get<CookieRowCounts>();\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,IAAa,kCAAkC;AAC/C,IAAa,+BAA+B;AAQ5C,IAAa,kCAA0C;kBACrC,gCAAgC;;;;;;;kBAOhC,6BAA6B;;;;;;;;AAS/C,IAAa,gCAAwC;0BAC3B,gCAAgC;0BAChC,6BAA6B;;;;;;;;;;;AA4DvD,SAAgB,UAAU,QAAkC;CAC1D,QAAQ,OAAO,KAAf;EACE,KAAK,yBACH,OAAO,CACL;GAAC,IAAI;GAAmB,OAAO,OAAO;GAAO,UAAU,OAAO;EAAG,CACnE;EAEF,KAAK,gBAAgB;GACnB,MAAM,EAAC,MAAM,UAAU,aAAY;GACnC,MAAM,QAAQ;IAAC,QAAQ,KAAK;IAAQ,MAAM,KAAK;GAAI;GACnD,MAAM,MAAkB,CAAC;GACzB,IAAI,UACF,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO;GAAQ,CAAC;GAEnD,IAAI,UACF,KAAK,MAAM,CAAC,QAAQ,OAAO,OAAO,QAAQ,QAAQ,GAChD,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO;IAAQ,UAAU;GAAE,CAAC;GAGjE,OAAO;EACT;EAEA,KAAK,gBACH,OAAO,CAAC;GAAC,IAAI;GAAgB,KAAK,OAAO;GAAK,KAAK,OAAO;EAAG,CAAC;EAEhE,KAAK,cACH,OAAO,CAAC;GAAC,IAAI;GAAc,OAAO,OAAO;EAAE,CAAC;EAE9C,KAAK,cAAc;GACjB,MAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;GACjD,MAAM,MAAkB,CAAC;GACzB,IAAI,eACF,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO,UAAU;GAAa,CAAC;GAElE,IAAI,UACF,IAAI,KAAK;IACP,IAAI;IACJ;IACA,QAAQ,OAAO;IACf;GACF,CAAC;GAEH,OAAO;EACT;EAEA,KAAK,iBAAiB;GAGpB,MAAM,EAAC,OAAO,KAAK,KAAK,YAAW;GACnC,OAAO,IAAI,SAAS,QAAQ,OACxB,CAAC,IACD,CAAC;IAAC,IAAI;IAAiB;IAAO,KAAK,IAAI;IAAM,KAAK,QAAQ;GAAI,CAAC;EACrE;EAEA,KAAK,eACH,OAAO,CAAC;GAAC,IAAI;GAAe,OAAO,OAAO;GAAO,QAAQ,OAAO;EAAM,CAAC;EAEzE,KAAK,sBAAsB;GACzB,MAAM,EAAC,UAAU,YAAW;GAG5B,MAAM,UAAU,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;GACvD,OAAO,QAAQ,WAAW,IACtB,CAAC,IACD,CACE;IACE,IAAI;IACJ,OAAO;KAAC,QAAQ,SAAS;KAAQ,MAAM,SAAS;IAAI;IACpD,SAAS;GACX,CACF;EACN;EAEA,KAAK;EACL,KAAK,cACH,OAAO,CAAC;EAEV,SAIE,YAAY,MAAM;CACtB;AACF;;;;;;;;;;;;;;AAeA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAKA,kBAAkB,GAAG,QAAgB;qBACzB,gCAAgC;;;;KAIhD;EACD,KAAKC,kBAAkB,GAAG,QAAgB;qBACzB,6BAA6B;;;;KAI7C;EACD,KAAKC,uBAAuB,GAAG,QAAgB;gBACnC,gCAAgC;;KAE3C;EACD,KAAKC,uBAAuB,GAAG,QAAgB;gBACnC,6BAA6B;;KAExC;EACD,KAAKC,uBAAuB,GAAG,QAAgB;qBAC9B,gCAAgC;;KAEhD;EACD,KAAKC,uBAAuB,GAAG,QAAgB;qBAC9B,6BAA6B;;KAE7C;EACD,KAAKC,wBAAwB,GAAG,QAAgB;gBACpC,6BAA6B;;KAExC;EACD,KAAKC,wBAAwB,GAAG,QAAgB;qBAC/B,6BAA6B;;KAE7C;CACH;;CAGA,MAAM,QAAkC;EACtC,MAAM,MAAM,UAAU,MAAM;EAC5B,KAAK,MAAM,MAAM,KACf,KAAKC,KAAK,EAAE;EAEd,OAAO;CACT;CAEA,KAAK,IAAoB;EACvB,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,KAAKR,gBAAgB,IACnB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,KAAKC,gBAAgB,IACnB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,QACH,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,KAAKC,qBAAqB,IACxB,GAAG,IAAI,QACP,GAAG,IAAI,MACP,GAAG,IAAI,QACP,GAAG,IAAI,IACT;IACA,KAAKC,qBAAqB,IACxB,GAAG,IAAI,QACP,GAAG,IAAI,MACP,GAAG,IAAI,QACP,GAAG,IAAI,IACT;IACA;GAEF,KAAK;IACH,KAAKC,qBAAqB,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IAC5D,KAAKC,qBAAqB,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IAC5D;GAEF,KAAK;IACH,KAAKC,sBAAsB,IACzB,GAAG,KACH,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,GACL;IACA;GAEF,KAAK;IACH,KAAKC,sBAAsB,IACzB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,MACL;IACA;GAEF,KAAK;IAIH,KAAK,MAAM,UAAU,GAAG,SACtB,KAAKA,sBAAsB,IACzB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,MACF;IAEF;GAEF,SACE,YAAY,EAAE;EAClB;CACF;AACF;AAEA,IAAM,YAAY,QAAgB,UAChC,KAAK,UAAU,CAAC,QAAQ,KAAK,CAAC;AAEhC,IAAM,aAAa,QAAgB,OAAe,WAChD,KAAK,UAAU;CAAC;CAAQ;CAAO;AAAM,CAAC;AAExC,IAAM,OAAO,GAAW,MAAe,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI;;;;;;;;;;;;;;;;AAiBhE,SAAgB,YACd,SACA,SACW;CACX,MAAM,WAAW,IAAI,IACnB,QAAQ,cAAc,KAAI,MAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CACjE;CACA,MAAM,cAAc,IAAI,IACtB,QAAQ,YAAY,KAAI,MAAK,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,CAC1E;CACA,MAAM,cAAc,EAAC,QAAQ,QAAmB,WAC9C,YAAY,OAAO,UAAU,QAAQ,MAAM,MAAM,CAAC;CAEpD,KAAK,MAAM,UAAU,SACnB,KAAK,MAAM,MAAM,UAAU,MAAM,GAC/B,QAAQ,GAAG,IAAX;EACE,KAAK;GACH,SAAS,IAAI,SAAS,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI,GAAG;IACrD,QAAQ,GAAG,MAAM;IACjB,OAAO,GAAG,MAAM;IAChB,UAAU,GAAG;GACf,CAAC;GACD;EAEF,KAAK;GACH,YAAY,IACV,UAAU,GAAG,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,MAAM,GACnD;IACE,QAAQ,GAAG,MAAM;IACjB,OAAO,GAAG,MAAM;IAChB,QAAQ,GAAG;IACX,UAAU,GAAG;GACf,CACF;GACA;EAEF,KAAK,gBAAgB;GACnB,MAAM,EAAC,KAAK,KAAK,YAAW;GAC5B,MAAM,QAAQ,SAAS,IAAI,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC;GACzD,IAAI,OAAO;IACT,SAAS,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC;IAC9C,SAAS,IAAI,SAAS,QAAQ,QAAQ,QAAQ,IAAI,GAAG;KACnD,GAAG;KACH,QAAQ,QAAQ;KAChB,OAAO,QAAQ;IACjB,CAAC;GACH;GAGA,KAAK,MAAM,CAAC,KAAK,WAAW,CAAC,GAAG,WAAW,GACzC,IAAI,OAAO,WAAW,IAAI,UAAU,OAAO,UAAU,IAAI,MAAM;IAC7D,YAAY,OAAO,GAAG;IACtB,YAAY,IACV,UAAU,QAAQ,QAAQ,QAAQ,MAAM,OAAO,MAAM,GACrD;KAAC,GAAG;KAAQ,QAAQ,QAAQ;KAAQ,OAAO,QAAQ;IAAI,CACzD;GACF;GAEF;EACF;EAEA,KAAK,cAAc;GACjB,MAAM,EAAC,QAAQ,SAAQ,GAAG;GAC1B,SAAS,OAAO,SAAS,QAAQ,IAAI,CAAC;GACtC,KAAK,MAAM,CAAC,KAAK,WAAW,CAAC,GAAG,WAAW,GACzC,IAAI,OAAO,WAAW,UAAU,OAAO,UAAU,MAC/C,YAAY,OAAO,GAAG;GAG1B;EACF;EAEA,KAAK,iBAAiB;GACpB,MAAM,EAAC,QAAQ,SAAQ,GAAG;GAC1B,MAAM,QAAQ,YAAY,IAAI,UAAU,QAAQ,MAAM,GAAG,GAAG,CAAC;GAC7D,IAAI,OAAO;IACT,YAAY,OAAO,UAAU,QAAQ,MAAM,GAAG,GAAG,CAAC;IAClD,YAAY,IAAI,UAAU,QAAQ,MAAM,GAAG,GAAG,GAAG;KAC/C,GAAG;KACH,QAAQ,GAAG;IACb,CAAC;GACH;GACA;EACF;EAEA,KAAK;GACH,WAAW,GAAG,OAAO,GAAG,MAAM;GAC9B;EAEF,KAAK;GACH,KAAK,MAAM,UAAU,GAAG,SACtB,WAAW,GAAG,OAAO,MAAM;GAE7B;EAEF,SACE,YAAY,EAAE;CAClB;CAIJ,OAAO;EACL,eAAe,CAAC,GAAG,SAAS,OAAO,CAAC,EAAE,UACnC,GAAG,MAAM,IAAI,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,EAAE,OAAO,EAAE,KAAK,CAC3D;EACA,aAAa,CAAC,GAAG,YAAY,OAAO,CAAC,EAAE,UACpC,GAAG,MACF,IAAI,EAAE,QAAQ,EAAE,MAAM,KACtB,IAAI,EAAE,OAAO,EAAE,KAAK,KACpB,IAAI,EAAE,QAAQ,EAAE,MAAM,CAC1B;CACF;AACF;AAEA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;AAM5D,SAAgB,qBAAqB,SAAuC;CAC1E,MAAM,WAAW,IAAI,IACnB,QAAQ,cAAc,KAAI,MAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,GAAG,EAAE,QAAQ,CAAC,CAC1E;CACA,MAAM,WAA8B,CAAC;CACrC,IAAI;CACJ,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,cAAa,QAAQ,aAAa;EACnE,IAAI,MAAM,MAAM,WAAW,UAAU,KAAK,MAAM,SAAS,OAAO;GAC9D,OAAO;IACL,OAAO;KACL;KACA,MAAM;KAGN,UAAU,SAAS,IAAI,SAAS,QAAQ,KAAK,CAAC,KAAK;IACrD;IACA,SAAS,CAAC;GACZ;GACA,SAAS,KAAK,IAAI;EACpB;EACA,KAAK,QAAQ,UAAU,EAAC,IAAI,SAAQ;CACtC;CACA,OAAO,MAAQ,UAAU,sBAAsB;AACjD;;;;;;AAOA,SAAgB,YAAY,IAAyB;CA4BnD,OAAO;EAAC,eA3Bc,GACnB,QAAgB;;gBAEL,gCAAgC;;KAE3C,EACA,IAAuD,EACvD,KAAK,EAAC,QAAQ,OAAO,gBAAe;GACnC;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAgBM;EAAe,aAdH,GACjB,QAAgB;;gBAEL,6BAA6B;;KAExC,EACA,IAAuE,EACvE,KAAK,EAAC,QAAQ,OAAO,QAAQ,gBAAe;GAC3C;GACA;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAEqB;CAAW;AACpC;;;;;;;;;;AAWA,SAAgB,eAAe,IAAc,SAA0B;CACrE,GAAG,QAAgB,gBAAgB,gCAAgC,EAAE,EAAE,IAAI;CAC3E,GAAG,QAAgB,gBAAgB,6BAA6B,EAAE,EAAE,IAAI;CAExE,IAAI,QAAQ,cAAc,SAAS,GAAG;EACpC,MAAM,SAAS,GAAG,QAAgB;qBACjB,gCAAgC;;KAEhD;EACD,KAAK,MAAM,EAAC,QAAQ,OAAO,cAAa,QAAQ,eAC9C,OAAO,IAAI,QAAQ,OAAO,WAAW,UAAU,QAAQ,CAAC;CAE5D;CACA,IAAI,QAAQ,YAAY,SAAS,GAAG;EAClC,MAAM,SAAS,GAAG,QAAgB;qBACjB,6BAA6B;;KAE7C;EACD,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,cAAa,QAAQ,aACtD,OAAO,IAAI,QAAQ,OAAO,QAAQ,WAAW,UAAU,QAAQ,CAAC;CAEpE;AACF;AAQA,SAAgB,oBAAoB,IAA+B;CACjE,OAAO,GACJ,QAAgB;;iCAEY,gCAAgC;;iCAEhC,6BAA6B;;KAEzD,EACA,IAAqB;AAC1B"}

package/out/zero-cache/src/services/replicator/change-processor.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-processor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;~~AAOjD~~,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAmB1E,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;~~AAiC5D~~,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAUpD,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG,cAAc,CAAC;AASlE,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,cAAc,GAAG,SAAS,CAAC;IAC9C,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3C,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,eAAe;;IAiB1B,YACE,EAAE,EAAE,eAAe,EACnB,IAAI,EAAE,mBAAmB,EACzB,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,KAAK,IAAI,EAQpD;IA2BD,KAAK,CAAC,EAAE,EAAE,UAAU,QAEnB;IAED,8CAA8C;IAC9C,cAAc,CACZ,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,gBAAgB,GAC3B,YAAY,GAAG,IAAI,CAkBrB;CAyIF"}
1	+ {"version":3,"file":"change-processor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAmB1E,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AAsC5D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAUpD,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG,cAAc,CAAC;AASlE,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,cAAc,GAAG,SAAS,CAAC;IAC9C,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3C,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,eAAe;;IAiB1B,YACE,EAAE,EAAE,eAAe,EACnB,IAAI,EAAE,mBAAmB,EACzB,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,KAAK,IAAI,EAQpD;IA2BD,KAAK,CAAC,EAAE,EAAE,UAAU,QAEnB;IAED,8CAA8C;IAC9C,cAAc,CACZ,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,gBAAgB,GAC3B,YAAY,GAAG,IAAI,CAkBrB;CAyIF"}

package/out/zero-cache/src/services/replicator/change-processor.js CHANGED Viewed

@@ -1,4 +1,5 @@
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
+import { deepEqual } from "../../../../shared/src/json.js";
 import { AbortError } from "../../../../shared/src/abort-error.js";
 import { must } from "../../../../shared/src/must.js";
 import { mapEntries } from "../../../../shared/src/objects.js";
@@ -15,6 +16,7 @@ import { ChangeLog } from "./schema/change-log.js";
 import { updateReplicationWatermark } from "./schema/replication-state.js";
 import { id } from "../../types/sql.js";
 import { createLiteIndexStatement, createLiteTableStatement, liteColumnDef } from "../../db/create.js";
+import { acceptBackfill } from "../change-source/protocol/backfill-progress.js";
 import { SqliteError } from "@rocicorp/zero-sqlite3";
 import { constants } from "node:buffer";
 //#region ../zero-cache/src/services/replicator/change-processor.ts
@@ -460,16 +462,18 @@ var TransactionProcessor = class {
 		}
 		this.#reloadTableSpecs();
 	}
-	processBackfill({ relation, watermark, columns, rowValues }) {
+	processBackfill({ relation, watermark, columns, rowValues, progressMarks }) {
 		const tableName = liteTableName(relation);
 		const tableSpec = must(this.#tableSpecs.get(tableName));
 		const rowKeyCols = relation.rowKey.columns;
 		const cols = [...rowKeyCols, ...columns];
-		const backfillingSet = new Set(tableSpec.backfilling ?? []);
-		if (!cols.some((c) => backfillingSet.has(c))) {
-			this.#lc.debug?.(`skipping redundant backfill of ${rowValues.length} rows into ${tableName} (all columns already published)`);
+		const { accepted, ignored } = this.#trackBackfillProgress(relation, cols, (own) => acceptBackfill(own, progressMarks));
+		if (accepted.length === 0) {
+			this.#lc.debug?.(ignored.length ? `skipping backfill of ${rowValues.length} rows into ${tableName}: not contiguous with the progress of columns [${ignored.join(",")}]` : `skipping redundant backfill of ${rowValues.length} rows into ${tableName} (all columns already published)`, progressMarks);
 			return;
 		}
+		if (ignored.length) this.#lc.debug?.(`backfill of ${tableName} is not contiguous with the progress of columns [${ignored.join(",")}]`, progressMarks);
+		const acceptedSet = new Set(accepted);
 		const insertColsStr = [...cols, ZERO_VERSION_COLUMN_NAME].map(id).join(",");
 		const qMarks = Array.from({ length: cols.length + 1 }).fill("?").join(",");
 		const rowKeyColsStr = rowKeyCols.map(id).join(",");
@@ -483,7 +487,7 @@ var TransactionProcessor = class {
 				skipped++;
 				continue;
 			}
-			const updates = (rowOp?.op === "s" ? cols.filter((c) => (rowOp.backfillingColumnVersions[c] ?? "") <= watermark) : cols).filter((c) => backfillingSet.has(c));
+			const updates = (rowOp?.op === "s" ? cols.filter((c) => (rowOp.backfillingColumnVersions[c] ?? "") <= watermark) : cols).filter((c) => acceptedSet.has(c));
 			if (updates.length === 0) {
 				skipped++;
 				continue;
@@ -499,19 +503,47 @@ var TransactionProcessor = class {
 		this.#lc.debug?.(`backfilled ${backfilled} rows (skipped ${skipped}) into ${tableName}`);
 	}
 	#completedBackfill;
+	/**
+	* Applies `accept` to the tracked progress of each of the `cols` that are
+	* still backfilling on this replica, updating the progress of the accepted
+	* ones. Columns that are not backfilling (e.g. already published) are
+	* neither accepted nor ignored.
+	*/
+	#trackBackfillProgress(relation, cols, accept) {
+		const tracked = this.#backfilling.getProgress(relation);
+		const accepted = [];
+		const ignored = [];
+		for (const col of cols) {
+			if (!tracked.has(col)) continue;
+			const progress = tracked.get(col);
+			const result = accept(progress);
+			if (!result.accept) {
+				ignored.push(col);
+				continue;
+			}
+			accepted.push(col);
+			if (!deepEqual(result.progress, progress)) this.#backfilling.setProgress(relation, col, result.progress);
+		}
+		return {
+			accepted,
+			ignored
+		};
+	}
 	processBackfillCompleted(msg) {
-		const { relation, columns, status } = msg;
+		const { relation, columns, status, progressMarks } = msg;
 		const tableName = liteTableName(relation);
-		const cols = [...relation.rowKey.columns, ...columns];
-		const tableSpec = must(this.#tableSpecs.get(tableName));
-		const backfillingSet = new Set(tableSpec.backfilling ?? []);
-		if (!cols.some((col) => backfillingSet.has(col))) {
-			this.#lc.debug?.(`skipping redundant backfill-completed for ${tableName} (all columns already published)`);
+		const rowKeyCols = relation.rowKey.columns;
+		const { accepted, ignored } = this.#trackBackfillProgress(relation, [...rowKeyCols, ...columns], (own) => acceptBackfill(own, progressMarks ? { previous: progressMarks.previous } : void 0));
+		if (ignored.length) this.#lc.info?.(`ignoring backfill-completed of ${tableName} for columns [${ignored.join(",")}], which have not received all of the data`, progressMarks);
+		if (accepted.length === 0) {
+			if (ignored.length === 0) this.#lc.debug?.(`skipping redundant backfill-completed for ${tableName} (all columns already published)`);
 			return;
 		}
+		const ignoredSet = new Set(ignored);
+		const cols = [...rowKeyCols.filter((c) => !ignoredSet.has(c)), ...columns.filter((c) => !ignoredSet.has(c))];
 		const columnMetadata = must(ColumnMetadataStore.getInstance(this.#db.db));
 		for (const col of cols) columnMetadata.clearBackfilling(tableName, col);
-		this.#backfilling.apply(msg);
+		this.#backfilling.complete(relation, cols);
 		this.#bumpVersions(relation);
 		if (status) this.#completedBackfill = {
 			table: tableName,

package/out/zero-cache/src/services/replicator/change-processor.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-processor.js","names":["#db","#changeLog","#tableMetadata","#backfilling","#mode","#failService","#tableSpecs","#failure","#currentTx","#fail","#processMessage","#beginTransaction","#lc","#startMs","#version","#jsonFormat","#columnMetadata","#reloadTableSpecs","#tableSpec","#upsert","#getKey","#logSetOp","#logDeleteOp","#delete","#logTruncateOp","#logResetOp","#bumpVersions","#pos","#numChangeLogEntries","#schemaChanged","#completedBackfill"],"sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"sourcesContent":["import {constants as bufferConstants} from 'node:buffer';\nimport type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {mapEntries} from '../../../../shared/src/objects.ts';\nimport type {DownloadStatus} from '../../../../zero-events/src/status.ts';\nimport {\n createLiteIndexStatement,\n createLiteTableStatement,\n liteColumnDef,\n} from '../../db/create.ts';\nimport {\n computeZqlSpecs,\n listTables,\n type LiteTableSpecWithReplicationStatus,\n} from '../../db/lite-tables.ts';\nimport {\n isArrayColumn,\n isEnumColumn,\n mapPostgresToLite,\n mapPostgresToLiteColumn,\n mapPostgresToLiteIndex,\n} from '../../db/pg-to-lite.ts';\nimport type {ColumnSpec} from '../../db/specs.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport type {LexiVersion} from '../../types/lexi-version.ts';\nimport {\n JSON_PARSED,\n liteRow,\n type JSONFormat,\n type LiteRow,\n type LiteRowKey,\n type LiteValueType,\n} from '../../types/lite.ts';\nimport {liteTableName} from '../../types/names.ts';\nimport {id} from '../../types/sql.ts';\nimport type {\n BackfillCompleted,\n Change,\n ColumnAdd,\n ColumnDrop,\n ColumnUpdate,\n Identifier,\n IndexCreate,\n IndexDrop,\n MessageBackfill,\n MessageCommit,\n MessageDelete,\n MessageInsert,\n MessageRelation,\n MessageTruncate,\n MessageUpdate,\n TableCreate,\n TableDrop,\n TableRename,\n TableUpdateMetadata,\n} from '../change-source/protocol/current/data.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from './replicator.ts';\nimport {BackfillingTracker} from './schema/backfilling.ts';\nimport {ChangeLog, DEL_OP, SET_OP} from './schema/change-log.ts';\nimport {ColumnMetadataStore} from './schema/column-metadata.ts';\nimport {\n ZERO_VERSION_COLUMN_NAME,\n updateReplicationWatermark,\n} from './schema/replication-state.ts';\nimport {TableMetadataTracker} from './schema/table-metadata.ts';\n\nexport type ChangeProcessorMode = ReplicatorMode \| 'initial-sync';\n\nconst BIND_VALUE_TOO_BIG_ERROR =\n 'The bound string, buffer, or bigint is too big';\nconst MAX_SQLITE_BIND_BYTES = Math.min(\n bufferConstants.MAX_LENGTH,\n bufferConstants.MAX_STRING_LENGTH,\n);\n\nexport type CommitResult = {\n watermark: string;\n completedBackfill: DownloadStatus \| undefined;\n schemaUpdated: boolean;\n changeLogUpdated: boolean;\n /*\n Millisecond epoch at which the transaction committed upstream, if the\n * ChangeSource reported one. Propagated to ViewSyncers as the origin\n * timestamp of the end-to-end serving lag measurement.\n /\n upstreamCommitTimeMs?: number \| undefined;\n};\n\n/\n The ChangeProcessor partitions the stream of messages into transactions\n * by creating a {@link TransactionProcessor} when a transaction begins, and dispatching\n * messages to it until the commit is received.\n \n From https://www.postgresql.org/docs/current/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW :\n \n \"The logical replication protocol sends individual transactions one by one.\n * This means that all messages between a pair of Begin and Commit messages\n * belong to the same transaction.\"\n /\nexport class ChangeProcessor {\n readonly #db: StatementRunner;\n readonly #changeLog: ChangeLog;\n readonly #tableMetadata: TableMetadataTracker;\n readonly #backfilling: BackfillingTracker;\n readonly #mode: ChangeProcessorMode;\n readonly #failService: (lc: LogContext, err: unknown) => void;\n\n // The TransactionProcessor lazily loads table specs into this Map,\n // and reloads them after a schema change. It is cached here to avoid\n // reading them from the DB on every transaction.\n readonly #tableSpecs = new Map<string, LiteTableSpecWithReplicationStatus>();\n\n #currentTx: TransactionProcessor \| null = null;\n\n #failure: Error \| undefined;\n\n constructor(\n db: StatementRunner,\n mode: ChangeProcessorMode,\n failService: (lc: LogContext, err: unknown) => void,\n ) {\n this.#db = db;\n this.#changeLog = new ChangeLog(db.db);\n this.#tableMetadata = new TableMetadataTracker(db.db);\n this.#backfilling = new BackfillingTracker(db.db);\n this.#mode = mode;\n this.#failService = failService;\n }\n\n #fail(lc: LogContext, err: unknown) {\n if (!this.#failure) {\n let failureError = err;\n const rollbackErrors = [attempt(() => this.#currentTx?.abort(lc))].filter(\n e => e !== undefined,\n );\n\n if (rollbackErrors.length) {\n const combinedError = new Error(\n `Message processing failed and rollback also failed: operation error = ${String(err)}; rollback error = ${rollbackErrors.map(String).join('; ')}`,\n );\n combinedError.cause = err;\n failureError = combinedError;\n }\n\n this.#failure = ensureError(failureError);\n\n if (!(this.#failure instanceof AbortError)) {\n // Propagate the failure up to the service.\n lc.error?.('Message Processing failed:', this.#failure);\n this.#failService(lc, this.#failure);\n }\n }\n }\n\n abort(lc: LogContext) {\n this.#fail(lc, new AbortError());\n }\n\n /* @return If a transaction was committed. /\n processMessage(\n lc: LogContext,\n downstream: ChangeStreamData,\n ): CommitResult \| null {\n const [type, message] = downstream;\n if (this.#failure) {\n lc.debug?.(`Dropping ${message.tag}`);\n return null;\n }\n try {\n const watermark =\n type === 'begin'\n ? downstream[2].commitWatermark\n : type === 'commit'\n ? downstream[2].watermark\n : undefined;\n return this.#processMessage(lc, message, watermark);\n } catch (e) {\n this.#fail(lc, e);\n }\n return null;\n }\n\n #beginTransaction(\n lc: LogContext,\n commitVersion: string,\n jsonFormat: JSONFormat,\n ): TransactionProcessor {\n const start = Date.now();\n\n // litestream can technically hold the lock for an arbitrary amount of time\n // when checkpointing a large commit. Crashing on the busy-timeout in this\n // scenario will either produce a corrupt backup or otherwise prevent\n // replication from proceeding.\n //\n // Instead, retry the lock acquisition indefinitely. If this masks\n // an unknown deadlock situation, manual intervention will be necessary.\n for (let i = 0; ; i++) {\n try {\n return new TransactionProcessor(\n lc,\n this.#db,\n this.#mode,\n this.#changeLog,\n this.#tableMetadata,\n this.#backfilling,\n this.#tableSpecs,\n commitVersion,\n jsonFormat,\n );\n } catch (e) {\n if (e instanceof SqliteError && e.code === 'SQLITE_BUSY') {\n lc.warn?.(\n `SQLITE_BUSY for ${Date.now() - start} ms (attempt ${i + 1}). ` +\n `This is only expected if litestream is performing a large ` +\n `checkpoint.`,\n e,\n );\n continue;\n }\n throw e;\n }\n }\n }\n\n /* @return If a transaction was committed. /\n #processMessage(\n lc: LogContext,\n msg: Change,\n watermark: string \| undefined,\n ): CommitResult \| null {\n if (msg.tag === 'begin') {\n if (this.#currentTx) {\n throw new Error(`Already in a transaction ${stringify(msg)}`);\n }\n this.#currentTx = this.#beginTransaction(\n lc,\n must(watermark),\n msg.json ?? JSON_PARSED,\n );\n return null;\n }\n\n // For non-begin messages, there should be a #currentTx set.\n const tx = this.#currentTx;\n if (!tx) {\n throw new Error(\n `Received message outside of transaction: ${stringify(msg)}`,\n );\n }\n\n if (msg.tag === 'commit') {\n assert(watermark, 'watermark is required for commit messages');\n const result = tx.processCommit(msg, watermark);\n // Clear only after a successful commit so #fail can roll back a commit\n // path that throws before SQLite has committed.\n this.#currentTx = null;\n return result;\n }\n\n if (msg.tag === 'rollback') {\n tx.abort(lc);\n this.#currentTx = null;\n return null;\n }\n\n switch (msg.tag) {\n case 'insert':\n tx.processInsert(msg);\n break;\n case 'update':\n tx.processUpdate(msg);\n break;\n case 'delete':\n tx.processDelete(msg);\n break;\n case 'truncate':\n tx.processTruncate(msg);\n break;\n case 'create-table':\n tx.processCreateTable(msg);\n break;\n case 'rename-table':\n tx.processRenameTable(msg);\n break;\n case 'update-table-metadata':\n tx.processTableMetadata(msg);\n break;\n case 'add-column':\n tx.processAddColumn(msg);\n break;\n case 'update-column':\n tx.processUpdateColumn(msg);\n break;\n case 'drop-column':\n tx.processDropColumn(msg);\n break;\n case 'drop-table':\n tx.processDropTable(msg);\n break;\n case 'create-index':\n tx.processCreateIndex(msg);\n break;\n case 'drop-index':\n tx.processDropIndex(msg);\n break;\n case 'backfill':\n tx.processBackfill(msg);\n break;\n case 'backfill-completed':\n tx.processBackfillCompleted(msg);\n break;\n default:\n unreachable(msg);\n }\n\n return null;\n }\n}\n\n/\n The {@link TransactionProcessor} handles the sequence of messages from\n * upstream, from `BEGIN` to `COMMIT` and executes the corresponding mutations\n * on the {@link postgres.TransactionSql} on the replica.\n \n When applying row contents to the replica, the `_0_version` column is added / updated,\n * and a corresponding entry in the `ChangeLog` is added. The version value is derived\n * from the watermark of the preceding transaction (stored as the `nextStateVersion` in the\n * `ReplicationState` table).\n \n Side note: For non-streaming Postgres transactions, the commitEndLsn (and thus\n * commit watermark) is available in the `begin` message, so it could theoretically\n * be used for the row version of changes within the transaction. However, the\n * commitEndLsn is not available in the streaming (in-progress) transaction\n * protocol, and may not be available for CDC streams of other upstream types.\n * Therefore, the zero replication protocol is designed to not require the commit\n * watermark when a transaction begins.\n \n Also of interest is the fact that all INSERT Messages are logically applied as\n * UPSERTs. See {@link processInsert} for the underlying motivation.\n /\nclass TransactionProcessor {\n readonly #lc: LogContext;\n readonly #startMs: number;\n readonly #db: StatementRunner;\n readonly #mode: ChangeProcessorMode;\n readonly #version: LexiVersion;\n readonly #changeLog: ChangeLog;\n readonly #tableMetadata: TableMetadataTracker;\n readonly #backfilling: BackfillingTracker;\n readonly #tableSpecs: Map<string, LiteTableSpecWithReplicationStatus>;\n readonly #jsonFormat: JSONFormat;\n readonly #columnMetadata: ColumnMetadataStore;\n\n #pos = 0;\n #schemaChanged = false;\n #numChangeLogEntries = 0;\n\n constructor(\n lc: LogContext,\n db: StatementRunner,\n mode: ChangeProcessorMode,\n changeLog: ChangeLog,\n tableMetadata: TableMetadataTracker,\n backfilling: BackfillingTracker,\n tableSpecs: Map<string, LiteTableSpecWithReplicationStatus>,\n commitVersion: LexiVersion,\n jsonFormat: JSONFormat,\n ) {\n this.#startMs = Date.now();\n this.#mode = mode;\n this.#jsonFormat = jsonFormat;\n\n switch (mode) {\n case 'serving':\n // This is the only transaction that commits to the serving replica.\n // Snapshotters use BEGIN CONCURRENT for private changes that are\n // always rolled back, while BEGIN IMMEDIATE lets this writer spill\n // dirty pages during large transactions.\n db.beginImmediate();\n break;\n case 'backup':\n // For the backup-replicator (i.e. replication-manager), there are no View Syncers\n // and thus BEGIN CONCURRENT is not necessary. In fact, BEGIN CONCURRENT can cause\n // deadlocks with forced wal-checkpoints (which `litestream replicate` performs),\n // so it is important to use vanilla transactions in this configuration.\n db.beginImmediate();\n break;\n case 'initial-sync':\n // When the ChangeProcessor is used for initial-sync, the calling code\n // handles the transaction boundaries.\n break;\n default:\n unreachable();\n }\n this.#db = db;\n this.#version = commitVersion;\n this.#lc = lc.withContext('version', commitVersion);\n this.#changeLog = changeLog;\n this.#tableMetadata = tableMetadata;\n this.#backfilling = backfilling;\n this.#tableSpecs = tableSpecs;\n // The column_metadata table is guaranteed to exist since the\n // replica-schema.ts migration to v8.\n this.#columnMetadata = must(ColumnMetadataStore.getInstance(db.db));\n\n if (this.#tableSpecs.size === 0) {\n this.#reloadTableSpecs();\n }\n }\n\n #reloadTableSpecs() {\n this.#tableSpecs.clear();\n // zqlSpecs include the primary key derived from unique indexes\n const zqlSpecs = computeZqlSpecs(this.#lc, this.#db.db, {\n includeBackfillingColumns: true,\n });\n for (let spec of listTables(this.#db.db)) {\n if (!spec.primaryKey) {\n spec = {\n ...spec,\n primaryKey: [\n ...(zqlSpecs.get(spec.name)?.tableSpec.primaryKey ?? []),\n ],\n };\n }\n this.#tableSpecs.set(spec.name, spec);\n }\n }\n\n #tableSpec(name: string) {\n return must(this.#tableSpecs.get(name), `Unknown table ${name}`);\n }\n\n #getKey(\n {row, numCols}: {row: LiteRow; numCols: number},\n {relation}: {relation: MessageRelation},\n ): LiteRowKey {\n const keyColumns =\n relation.rowKey.type !== 'full'\n ? relation.rowKey.columns // already a suitable key\n : this.#tableSpec(liteTableName(relation)).primaryKey;\n if (!keyColumns?.length) {\n throw new Error(\n `Cannot replicate table \"${relation.name}\" without a PRIMARY KEY or UNIQUE INDEX`,\n );\n }\n // For the common case (replica identity default), the row is already the\n // key for deletes and updates, in which case a new object can be avoided.\n if (numCols === keyColumns.length) {\n return row;\n }\n const key: Record<string, LiteValueType> = {};\n for (const col of keyColumns) {\n key[col] = row[col];\n }\n return key;\n }\n\n processInsert(insert: MessageInsert) {\n const table = liteTableName(insert.relation);\n const tableSpec = this.#tableSpec(table);\n const newRow = liteRow(insert.new, tableSpec, this.#jsonFormat);\n\n this.#upsert(table, {\n ...newRow.row,\n [ZERO_VERSION_COLUMN_NAME]: this.#version,\n });\n\n if (insert.relation.rowKey.columns.length === 0) {\n // INSERTs can be replicated for rows without a PRIMARY KEY or a\n // UNIQUE INDEX. These are written to the replica but not recorded\n // in the changeLog, because these rows cannot participate in IVM.\n //\n // (Once the table schema has been corrected to include a key, the\n // associated schema change will reset pipelines and data can be\n // loaded via hydration.)\n return;\n }\n const key = this.#getKey(newRow, insert);\n this.#logSetOp(table, key, getBackfilledColumns(newRow.row, tableSpec));\n }\n\n #upsert(table: string, row: LiteRow) {\n const columns = Object.keys(row).map(c => id(c));\n this.#db.run(\n `\n INSERT OR REPLACE INTO ${id(table)} (${columns.join(',')})\n VALUES (${Array.from({length: columns.length}).fill('?').join(',')})\n `,\n Object.values(row),\n );\n }\n\n // Updates by default are applied as UPDATE commands to support partial\n // row specifications from the change source. In particular, this is needed\n // to handle updates for which unchanged TOASTed values are not sent:\n //\n // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html#PROTOCOL-LOGICALREP-MESSAGE-FORMATS-TUPLEDATA\n //\n // However, in certain cases an UPDATE may be received for a row that\n // was not initially synced, such as when, an existing table is added\n // to the app's publication.\n //\n // In order to facilitate \"resumptive\" replication, the logic falls back to\n // an INSERT if the update did not change any rows.\n processUpdate(update: MessageUpdate) {\n const table = liteTableName(update.relation);\n const tableSpec = this.#tableSpec(table);\n const newRow = liteRow(update.new, tableSpec, this.#jsonFormat);\n const row = {...newRow.row, [ZERO_VERSION_COLUMN_NAME]: this.#version};\n\n // update.key is set with the old values if the key has changed.\n const oldKey = update.key\n ? this.#getKey(\n liteRow(update.key, this.#tableSpec(table), this.#jsonFormat),\n update,\n )\n : null;\n const newKey = this.#getKey(newRow, update);\n\n if (oldKey) {\n this.#logDeleteOp(table, oldKey, tableSpec.backfilling);\n }\n this.#logSetOp(table, newKey, getBackfilledColumns(newRow.row, tableSpec));\n\n const currKey = oldKey ?? newKey;\n const conds = Object.keys(currKey).map(col => `${id(col)}=?`);\n const setExprs = Object.keys(row).map(col => `${id(col)}=?`);\n\n try {\n const {changes} = this.#db.run(\n `\n UPDATE ${id(table)}\n SET ${setExprs.join(',')}\n WHERE ${conds.join(' AND ')}\n `,\n [...Object.values(row), ...Object.values(currKey)],\n );\n\n // If the UPDATE did not affect any rows, perform an UPSERT of the\n // new row for resumptive replication.\n if (changes === 0) {\n this.#upsert(table, row);\n }\n } catch (e) {\n if (\n !(e instanceof RangeError) \|\|\n e.message !== BIND_VALUE_TOO_BIG_ERROR\n ) {\n throw e;\n }\n\n const binding = [...Object.entries(row), ...Object.entries(currKey)]\n .map(\n ([column, value]) =>\n [column, oversizedValueDescription(value)] as const,\n )\n .find(([, description]) => description);\n const relationOid =\n 'relationOid' in update.relation &&\n typeof update.relation.relationOid === 'number'\n ? ` relationOid=${update.relation.relationOid}`\n : '';\n const error = new Error(\n `Oversized SQLite update binding: tx=${this.#version}${relationOid} table=${update.relation.schema}.${update.relation.name} column=${binding?.[0] ?? 'unknown'}${binding?.[1] ? ` ${binding[1]}` : ''}`,\n {cause: e},\n );\n error.name = 'OversizedUpdateBindingError';\n throw error;\n }\n }\n\n processDelete(del: MessageDelete) {\n const table = liteTableName(del.relation);\n const tableSpec = this.#tableSpec(table);\n const rowKey = this.#getKey(\n liteRow(del.key, tableSpec, this.#jsonFormat),\n del,\n );\n\n this.#delete(table, rowKey);\n this.#logDeleteOp(table, rowKey, tableSpec.backfilling);\n }\n\n #delete(table: string, rowKey: LiteRowKey) {\n const conds = Object.keys(rowKey).map(col => `${id(col)}=?`);\n this.#db.run(\n `DELETE FROM ${id(table)} WHERE ${conds.join(' AND ')}`,\n Object.values(rowKey),\n );\n }\n\n processTruncate(truncate: MessageTruncate) {\n for (const relation of truncate.relations) {\n const table = liteTableName(relation);\n // Update replica data.\n this.#db.run(`DELETE FROM ${id(table)}`);\n\n // Update change log.\n this.#logTruncateOp(table);\n }\n }\n\n processCreateTable(create: TableCreate) {\n if (create.metadata) {\n this.#tableMetadata.setUpstreamMetadata(create.spec, create.metadata);\n }\n this.#backfilling.apply(create);\n const table = mapPostgresToLite(create.spec);\n this.#db.db.exec(createLiteTableStatement(table));\n\n // Write to metadata table\n for (const [colName, colSpec] of Object.entries(create.spec.columns)) {\n this.#columnMetadata.insert(\n table.name,\n colName,\n colSpec,\n create.backfill?.[colName],\n );\n }\n\n this.#logResetOp(table.name);\n this.#lc.info?.(create.tag, table.name);\n }\n\n processTableMetadata(msg: TableUpdateMetadata) {\n this.#tableMetadata.setUpstreamMetadata(msg.table, msg.new);\n // Inert today: the fold's only op for this tag is `upsert-metadata`, which\n // the line above is the replica's interpreter of. Called anyway so that\n // every site that hands a schema change to one cookie tracker hands it to\n // both -- an exception here would be a schema change the shared fold has an\n // opinion about and this store never sees, which is the drift the fold\n // exists to make impossible.\n this.#backfilling.apply(msg);\n }\n\n processRenameTable(rename: TableRename) {\n this.#tableMetadata.rename(rename.old, rename.new);\n this.#backfilling.apply(rename);\n\n const oldName = liteTableName(rename.old);\n const newName = liteTableName(rename.new);\n this.#db.db.exec(`ALTER TABLE ${id(oldName)} RENAME TO ${id(newName)}`);\n\n // Rename in metadata table\n this.#columnMetadata.renameTable(oldName, newName);\n\n this.#bumpVersions(rename.new);\n this.#logResetOp(oldName);\n this.#lc.info?.(rename.tag, oldName, newName);\n }\n\n processAddColumn(msg: ColumnAdd) {\n if (msg.tableMetadata) {\n this.#tableMetadata.setUpstreamMetadata(msg.table, msg.tableMetadata);\n }\n this.#backfilling.apply(msg);\n const table = liteTableName(msg.table);\n const {name} = msg.column;\n const spec = mapPostgresToLiteColumn(table, msg.column);\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} ADD ${id(name)} ${liteColumnDef(spec)}`,\n );\n\n // Write to metadata table\n this.#columnMetadata.insert(table, name, msg.column.spec, msg.backfill);\n\n if (msg.backfill) {\n this.#reloadTableSpecs();\n } else {\n // Make the new column visible immediately if it's not being backfilled.\n // Otherwise, the version bump will happen with the backfill is complete.\n this.#bumpVersions(msg.table);\n }\n this.#lc.info?.(msg.tag, table, msg.column);\n }\n\n processUpdateColumn(msg: ColumnUpdate) {\n // A no-op unless this is a rename, which is the only part of a column\n // update that moves the backfill cookie.\n this.#backfilling.apply(msg);\n\n const table = liteTableName(msg.table);\n let oldName = msg.old.name;\n const newName = msg.new.name;\n\n const storageTypesDiffer = differentStorageTypes(\n msg.old.spec,\n msg.new.spec,\n );\n\n // If neither the column name nor the SQLite data type changes, only the\n // upstream metadata needs to be updated. This includes changes such as a\n // varchar character limit and nullability, which SQLite does not enforce\n // but a freshly built replica still records.\n if (oldName === newName && !storageTypesDiffer) {\n this.#columnMetadata.update(\n table,\n msg.old.name,\n msg.new.name,\n msg.new.spec,\n );\n if (Boolean(msg.old.spec.notNull) !== Boolean(msg.new.spec.notNull)) {\n this.#bumpVersions(msg.table);\n }\n this.#lc.info?.(msg.tag, 'updated metadata only', msg.old, msg.new);\n return;\n }\n // Storage type changes require a new column so that SQLite uses the new\n // representation for existing and future values.\n if (storageTypesDiffer) {\n // update-column can ignore defaults because it does not change the values\n // in existing rows.\n //\n // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DESC-SET-DROP-DEFAULT\n //\n // \"The new default value will only apply in subsequent INSERT or UPDATE\n // commands; it does not cause rows already in the table to change.\"\n //\n // This allows support for _changing_ column defaults to any expression,\n // since it does not affect what the replica needs to do.\n const newLiteSpec = mapPostgresToLiteColumn(\n table,\n msg.new,\n 'ignore-default',\n );\n if (oldName !== newName) {\n // Rename the column in place first so that SQLite rewrites every\n // reference to it in the table's index definitions, including the\n // WHERE clauses of partial indexes.\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} RENAME ${id(oldName)} TO ${id(newName)}`,\n );\n oldName = newName;\n }\n const tableSpec = must(\n listTables(this.#db.db, false, false).find(\n tableSpec => tableSpec.name === table,\n ),\n );\n // Recreating the table drops its indexes, so capture their DDL to\n // replay afterwards. Auto-indexes (PRIMARY KEY) have no DDL and are\n // recreated by the CREATE TABLE statement itself.\n const indexDDL = this.#db.db\n .prepare(\n `SELECT sql FROM sqlite_master\n WHERE type = 'index' AND tbl_name = ? AND sql IS NOT NULL\n ORDER BY rowid`,\n )\n .all<{sql: string}>(table)\n .map(({sql}) => sql);\n const tmpTable = `tmp.${table}`;\n const columns = Object.keys(tableSpec.columns);\n const stmts = [\n createLiteTableStatement({\n ...tableSpec,\n name: tmpTable,\n columns: mapEntries(tableSpec.columns, (column, spec) => [\n column,\n column === newName ? {...newLiteSpec, pos: spec.pos} : spec,\n ]),\n }),\n `INSERT INTO ${id(tmpTable)} (${columns.map(id).join(',')})\n SELECT ${columns.map(id).join(',')} FROM ${id(table)};`,\n `DROP TABLE ${id(table)};`,\n `ALTER TABLE ${id(tmpTable)} RENAME TO ${id(table)};`,\n ...indexDDL.map(sql => `${sql};`),\n ];\n this.#db.db.exec(stmts.join(''));\n }\n if (oldName !== newName) {\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} RENAME ${id(oldName)} TO ${id(newName)}`,\n );\n }\n\n // Update metadata table\n this.#columnMetadata.update(\n table,\n msg.old.name,\n msg.new.name,\n msg.new.spec,\n );\n\n this.#bumpVersions(msg.table);\n this.#lc.info?.(msg.tag, table, msg.new);\n }\n\n processDropColumn(msg: ColumnDrop) {\n const table = liteTableName(msg.table);\n const {column} = msg;\n this.#db.db.exec(`ALTER TABLE ${id(table)} DROP ${id(column)}`);\n\n // Delete from metadata table\n this.#columnMetadata.deleteColumn(table, column);\n this.#backfilling.apply(msg);\n\n this.#bumpVersions(msg.table);\n this.#lc.info?.(msg.tag, table, column);\n }\n\n processDropTable(drop: TableDrop) {\n this.#tableMetadata.drop(drop.id);\n this.#backfilling.apply(drop);\n\n const name = liteTableName(drop.id);\n this.#db.db.exec(`DROP TABLE IF EXISTS ${id(name)}`);\n\n // Delete from metadata table\n this.#columnMetadata.deleteTable(name);\n\n this.#logResetOp(name);\n this.#lc.info?.(drop.tag, name);\n }\n\n processCreateIndex(create: IndexCreate) {\n const index = mapPostgresToLiteIndex(create.spec);\n this.#db.db.exec(createLiteIndexStatement(index));\n\n // indexes affect tables visibility (e.g. sync-ability is gated on\n // having a unique index), so reset pipelines to refresh table schemas.\n this.#logResetOp(index.tableName);\n this.#lc.info?.(create.tag, index.name);\n }\n\n processDropIndex(drop: IndexDrop) {\n const name = liteTableName(drop.id);\n this.#db.db.exec(`DROP INDEX IF EXISTS ${id(name)}`);\n this.#lc.info?.(drop.tag, name);\n }\n\n #bumpVersions(table: Identifier) {\n this.#tableMetadata.setMinRowVersion(table, this.#version);\n this.#logResetOp(liteTableName(table));\n }\n\n /\n @param backfilledColumns `backfilling` columns for which values were set\n /\n #logSetOp(\n table: string,\n key: LiteRowKey,\n backfilledColumns: string[] \| undefined,\n ) {\n // The \"serving\" replicator always writes to the change-log (for IVM).\n // The \"backup\" replicator only needs to write to the change log\n // when writing columns that are being backfilled.\n if (this.#mode === 'serving' \|\| backfilledColumns !== undefined) {\n this.#changeLog.logSetOp(\n this.#version,\n this.#pos++,\n table,\n key,\n backfilledColumns,\n );\n this.#numChangeLogEntries++;\n }\n }\n\n #logDeleteOp(table: string, key: LiteRowKey, backfilling?: string[]) {\n // The \"serving\" replicator always writes to the change-log (for IVM).\n // The \"backup\" replicator only needs to write to the change log\n // when writing columns that are being backfilled.\n if (this.#mode === 'serving' \|\| backfilling?.length) {\n this.#changeLog.logDeleteOp(this.#version, this.#pos++, table, key);\n this.#numChangeLogEntries++;\n }\n }\n\n #logTruncateOp(table: string) {\n if (this.#mode === 'serving') {\n this.#changeLog.logTruncateOp(this.#version, table);\n this.#numChangeLogEntries++;\n }\n }\n\n #logResetOp(table: string) {\n this.#schemaChanged = true;\n if (this.#mode === 'serving') {\n this.#changeLog.logResetOp(this.#version, table);\n this.#numChangeLogEntries++;\n }\n this.#reloadTableSpecs();\n }\n\n processBackfill({relation, watermark, columns, rowValues}: MessageBackfill) {\n const tableName = liteTableName(relation);\n const tableSpec = must(this.#tableSpecs.get(tableName));\n const rowKeyCols = relation.rowKey.columns;\n const cols = [...rowKeyCols, ...columns];\n\n // Columns that are still being backfilled on _this_ replica. Once a column\n // has been published (backfill completed), its values strictly follow the\n // replication timeline, so a subsequent backfill must never overwrite\n // them. This guards against a redundant backfill, e.g. when the\n // view-syncer (re)connects to a replication-manager that is (re)running\n // a backfill this replica already completed.\n const backfillingSet = new Set(tableSpec.backfilling ?? []);\n if (!cols.some(c => backfillingSet.has(c))) {\n // Every delivered column is already published: the backfill is entirely\n // redundant. Skip it rather than rewind published values to the snapshot.\n this.#lc.debug?.(\n `skipping redundant backfill of ${rowValues.length} rows into ` +\n `${tableName} (all columns already published)`,\n );\n return;\n }\n\n // Common parts of the INSERT sql statement.\n const insertColsStr = [...cols, ZERO_VERSION_COLUMN_NAME].map(id).join(',');\n const qMarks = Array.from({length: cols.length + 1})\n .fill('?')\n .join(',');\n const rowKeyColsStr = rowKeyCols.map(id).join(',');\n\n let backfilled = 0;\n let skipped = 0;\n for (const v of rowValues) {\n const row = liteRow(\n Object.fromEntries(cols.map((c, i) => [c, v[i]])),\n tableSpec,\n this.#jsonFormat,\n );\n const rowKey = this.#getKey(row, {relation});\n const rowOp = this.#changeLog.getLatestRowOp(tableName, rowKey);\n if (rowOp?.op === DEL_OP && rowOp.stateVersion > watermark) {\n skipped++;\n continue; // the row was deleted after the backfill snapshot\n }\n // Restrict the DO UPDATE columns to those still being backfilled. Never\n // update an already backfilled (i.e. published) column.\n const updates = (\n rowOp?.op === SET_OP\n ? cols.filter(\n c => (rowOp.backfillingColumnVersions[c] ?? '') <= watermark,\n )\n : cols\n ).filter(c => backfillingSet.has(c));\n if (updates.length === 0) {\n // row already has newer (or published) values for all backfilling\n // columns.\n skipped++;\n continue;\n }\n const updateStmts = updates.map(col => `${id(col)}=excluded.${id(col)}`);\n this.#db.run(\n /sql/ `\n INSERT INTO ${id(tableName)} (${insertColsStr}) VALUES (${qMarks})\n ON CONFLICT (${rowKeyColsStr})\n DO UPDATE SET ${updateStmts.join(',')};\n `,\n ...Object.values(row.row),\n watermark, // the _0_version for new rows (i.e. table backfill)\n );\n backfilled++;\n }\n\n this.#lc.debug?.(\n `backfilled ${backfilled} rows (skipped ${skipped}) into ${tableName}`,\n );\n }\n\n #completedBackfill: DownloadStatus \| undefined;\n\n processBackfillCompleted(msg: BackfillCompleted) {\n const {relation, columns, status} = msg;\n const tableName = liteTableName(relation);\n const rowKeyCols = relation.rowKey.columns;\n const cols = [...rowKeyCols, ...columns];\n\n // If none of the columns are still backfilling on this replica, they were\n // already published (e.g. a redundant backfill from a replication-manager\n // re-running a backfill this replica already completed). Bumping versions\n // again would spuriously reset pipelines and re-report completion, so treat\n // the completion as a no-op.\n const tableSpec = must(this.#tableSpecs.get(tableName));\n const backfillingSet = new Set(tableSpec.backfilling ?? []);\n if (!cols.some(col => backfillingSet.has(col))) {\n this.#lc.debug?.(\n `skipping redundant backfill-completed for ${tableName} ` +\n `(all columns already published)`,\n );\n return;\n }\n\n const columnMetadata = must(ColumnMetadataStore.getInstance(this.#db.db));\n for (const col of cols) {\n columnMetadata.clearBackfilling(tableName, col);\n }\n this.#backfilling.apply(msg);\n // Given that new columns are being exposed for every row in the table, bump the\n // row version for all rows.\n this.#bumpVersions(relation);\n if (status) {\n this.#completedBackfill = {table: tableName, columns: cols, ...status};\n }\n this.#lc.info?.(`finished backfilling ${tableName}`);\n\n // Note that there is no need to clear the backfillingColumnVersions values\n // in the changeLog. It could theoretically be done for clarity but:\n // (1) it could be non-trivial in terms of latency introduced and\n // (2) the data must be preserved if _other_ columns are in the process\n // of being backfilled\n //\n // Thus, for speed and simplicity, the values are left as is. (Note that\n // subsequent replicated changes to those rows will clear the values if\n // no backfills are in progress).\n }\n\n processCommit(commit: MessageCommit, watermark: string): CommitResult {\n if (watermark !== this.#version) {\n throw new Error(\n `'commit' version ${watermark} does not match 'begin' version ${\n this.#version\n }: ${stringify(commit)}`,\n );\n }\n updateReplicationWatermark(this.#db, watermark);\n\n if (this.#schemaChanged) {\n const start = Date.now();\n this.#db.db.pragma('optimize');\n this.#lc.info?.(\n `PRAGMA optimized after schema change (${Date.now() - start} ms)`,\n );\n }\n\n if (this.#mode !== 'initial-sync') {\n this.#db.commit();\n }\n\n const elapsedMs = Date.now() - this.#startMs;\n this.#lc.debug?.(`Committed tx@${this.#version} (${elapsedMs} ms)`);\n\n return {\n watermark,\n completedBackfill: this.#completedBackfill,\n schemaUpdated: this.#schemaChanged,\n changeLogUpdated: this.#numChangeLogEntries > 0,\n upstreamCommitTimeMs: commit.commitTimeMs,\n };\n }\n\n abort(lc: LogContext) {\n lc.info?.(`aborting transaction ${this.#version}`);\n this.#db.rollback();\n // A schema change in the transaction reloaded the table specs, which now\n // describe the schema that the rollback undid. Clearing them makes the\n // next transaction reload them from the database.\n this.#tableSpecs.clear();\n }\n}\n\nfunction differentStorageTypes(\n oldSpec: ColumnSpec,\n newSpec: ColumnSpec,\n): boolean {\n return (\n oldSpec.dataType !== newSpec.dataType \|\|\n isEnumColumn(oldSpec) !== isEnumColumn(newSpec) \|\|\n isArrayColumn(oldSpec) !== isArrayColumn(newSpec)\n );\n}\n\nfunction getBackfilledColumns(\n row: LiteRow,\n {backfilling}: LiteTableSpecWithReplicationStatus,\n): string[] \| undefined {\n if (!backfilling?.length) {\n return undefined; // common case\n }\n return backfilling.filter(col => col in row);\n}\n\nfunction oversizedValueDescription(value: LiteValueType): string \| undefined {\n if (typeof value === 'bigint') {\n return BigInt.asIntN(64, value) !== value\n ? 'valueType=bigint fitsInt64=false'\n : undefined;\n }\n\n const sizeBytes =\n typeof value === 'string'\n ? Buffer.byteLength(value)\n : value instanceof Uint8Array\n ? value.byteLength\n : undefined;\n if (sizeBytes !== undefined && sizeBytes > MAX_SQLITE_BIND_BYTES) {\n const valueType = typeof value === 'string' ? 'string' : 'buffer';\n return `valueType=${valueType} sizeBytes=${sizeBytes} limitBytes=${MAX_SQLITE_BIND_BYTES}`;\n }\n\n return undefined;\n}\n\n/* Runs `fn`, returning its error rather than throwing it. */\nfunction attempt(fn: () => void): unknown {\n try {\n fn();\n return undefined;\n } catch (e) {\n return e ?? new Error('rollback failed');\n }\n}\n\nfunction ensureError(err: unknown): Error {\n if (err instanceof Error) {\n return err;\n }\n const error = new Error();\n error.cause = err;\n return error;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyEA,IAAM,2BACJ;AACF,IAAM,wBAAwB,KAAK,IACjC,UAAgB,YAChB,UAAgB,iBAClB;;;;;;;;;;;;AA0BA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CACA;CACA;CAKA,8BAAuB,IAAI,IAAgD;CAE3E,aAA0C;CAE1C;CAEA,YACE,IACA,MACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,aAAa,IAAI,UAAU,GAAG,EAAE;EACrC,KAAKC,iBAAiB,IAAI,qBAAqB,GAAG,EAAE;EACpD,KAAKC,eAAe,IAAI,mBAAmB,GAAG,EAAE;EAChD,KAAKC,QAAQ;EACb,KAAKC,eAAe;CACtB;CAEA,MAAM,IAAgB,KAAc;EAClC,IAAI,CAAC,KAAKE,UAAU;GAClB,IAAI,eAAe;GACnB,MAAM,iBAAiB,CAAC,cAAc,KAAKC,YAAY,MAAM,EAAE,CAAC,CAAC,EAAE,QACjE,MAAK,MAAM,KAAA,CACb;GAEA,IAAI,eAAe,QAAQ;IACzB,MAAM,gCAAgB,IAAI,MACxB,yEAAyE,OAAO,GAAG,EAAE,qBAAqB,eAAe,IAAI,MAAM,EAAE,KAAK,IAAI,GAChJ;IACA,cAAc,QAAQ;IACtB,eAAe;GACjB;GAEA,KAAKD,WAAW,YAAY,YAAY;GAExC,IAAI,EAAE,KAAKA,oBAAoB,aAAa;IAE1C,GAAG,QAAQ,8BAA8B,KAAKA,QAAQ;IACtD,KAAKF,aAAa,IAAI,KAAKE,QAAQ;GACrC;EACF;CACF;CAEA,MAAM,IAAgB;EACpB,KAAKE,MAAM,IAAI,IAAI,WAAW,CAAC;CACjC;;CAGA,eACE,IACA,YACqB;EACrB,MAAM,CAAC,MAAM,WAAW;EACxB,IAAI,KAAKF,UAAU;GACjB,GAAG,QAAQ,YAAY,QAAQ,KAAK;GACpC,OAAO;EACT;EACA,IAAI;GACF,MAAM,YACJ,SAAS,UACL,WAAW,GAAG,kBACd,SAAS,WACP,WAAW,GAAG,YACd,KAAA;GACR,OAAO,KAAKG,gBAAgB,IAAI,SAAS,SAAS;EACpD,SAAS,GAAG;GACV,KAAKD,MAAM,IAAI,CAAC;EAClB;EACA,OAAO;CACT;CAEA,kBACE,IACA,eACA,YACsB;EACtB,MAAM,QAAQ,KAAK,IAAI;EASvB,KAAK,IAAI,IAAI,IAAK,KAChB,IAAI;GACF,OAAO,IAAI,qBACT,IACA,KAAKT,KACL,KAAKI,OACL,KAAKH,YACL,KAAKC,gBACL,KAAKC,cACL,KAAKG,aACL,eACA,UACF;EACF,SAAS,GAAG;GACV,IAAI,aAAa,eAAe,EAAE,SAAS,eAAe;IACxD,GAAG,OACD,mBAAmB,KAAK,IAAI,IAAI,MAAM,eAAe,IAAI,EAAE,2EAG3D,CACF;IACA;GACF;GACA,MAAM;EACR;CAEJ;;CAGA,gBACE,IACA,KACA,WACqB;EACrB,IAAI,IAAI,QAAQ,SAAS;GACvB,IAAI,KAAKE,YACP,MAAM,IAAI,MAAM,4BAA4B,UAAU,GAAG,GAAG;GAE9D,KAAKA,aAAa,KAAKG,kBACrB,IACA,KAAK,SAAS,GACd,IAAI,QAAA,GACN;GACA,OAAO;EACT;EAGA,MAAM,KAAK,KAAKH;EAChB,IAAI,CAAC,IACH,MAAM,IAAI,MACR,4CAA4C,UAAU,GAAG,GAC3D;EAGF,IAAI,IAAI,QAAQ,UAAU;GACxB,OAAO,WAAW,2CAA2C;GAC7D,MAAM,SAAS,GAAG,cAAc,KAAK,SAAS;GAG9C,KAAKA,aAAa;GAClB,OAAO;EACT;EAEA,IAAI,IAAI,QAAQ,YAAY;GAC1B,GAAG,MAAM,EAAE;GACX,KAAKA,aAAa;GAClB,OAAO;EACT;EAEA,QAAQ,IAAI,KAAZ;GACE,KAAK;IACH,GAAG,cAAc,GAAG;IACpB;GACF,KAAK;IACH,GAAG,cAAc,GAAG;IACpB;GACF,KAAK;IACH,GAAG,cAAc,GAAG;IACpB;GACF,KAAK;IACH,GAAG,gBAAgB,GAAG;IACtB;GACF,KAAK;IACH,GAAG,mBAAmB,GAAG;IACzB;GACF,KAAK;IACH,GAAG,mBAAmB,GAAG;IACzB;GACF,KAAK;IACH,GAAG,qBAAqB,GAAG;IAC3B;GACF,KAAK;IACH,GAAG,iBAAiB,GAAG;IACvB;GACF,KAAK;IACH,GAAG,oBAAoB,GAAG;IAC1B;GACF,KAAK;IACH,GAAG,kBAAkB,GAAG;IACxB;GACF,KAAK;IACH,GAAG,iBAAiB,GAAG;IACvB;GACF,KAAK;IACH,GAAG,mBAAmB,GAAG;IACzB;GACF,KAAK;IACH,GAAG,iBAAiB,GAAG;IACvB;GACF,KAAK;IACH,GAAG,gBAAgB,GAAG;IACtB;GACF,KAAK;IACH,GAAG,yBAAyB,GAAG;IAC/B;GACF,SACE,YAAY,GAAG;EACnB;EAEA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,OAAO;CACP,iBAAiB;CACjB,uBAAuB;CAEvB,YACE,IACA,IACA,MACA,WACA,eACA,aACA,YACA,eACA,YACA;EACA,KAAKK,WAAW,KAAK,IAAI;EACzB,KAAKT,QAAQ;EACb,KAAKW,cAAc;EAEnB,QAAQ,MAAR;GACE,KAAK;IAKH,GAAG,eAAe;IAClB;GACF,KAAK;IAKH,GAAG,eAAe;IAClB;GACF,KAAK,gBAGH;GACF,SACE,YAAY;EAChB;EACA,KAAKf,MAAM;EACX,KAAKc,WAAW;EAChB,KAAKF,MAAM,GAAG,YAAY,WAAW,aAAa;EAClD,KAAKX,aAAa;EAClB,KAAKC,iBAAiB;EACtB,KAAKC,eAAe;EACpB,KAAKG,cAAc;EAGnB,KAAKU,kBAAkB,KAAK,oBAAoB,YAAY,GAAG,EAAE,CAAC;EAElE,IAAI,KAAKV,YAAY,SAAS,GAC5B,KAAKW,kBAAkB;CAE3B;CAEA,oBAAoB;EAClB,KAAKX,YAAY,MAAM;EAEvB,MAAM,WAAW,gBAAgB,KAAKM,KAAK,KAAKZ,IAAI,IAAI,EACtD,2BAA2B,KAC7B,CAAC;EACD,KAAK,IAAI,QAAQ,WAAW,KAAKA,IAAI,EAAE,GAAG;GACxC,IAAI,CAAC,KAAK,YACR,OAAO;IACL,GAAG;IACH,YAAY,CACV,GAAI,SAAS,IAAI,KAAK,IAAI,GAAG,UAAU,cAAc,CAAC,CACxD;GACF;GAEF,KAAKM,YAAY,IAAI,KAAK,MAAM,IAAI;EACtC;CACF;CAEA,WAAW,MAAc;EACvB,OAAO,KAAK,KAAKA,YAAY,IAAI,IAAI,GAAG,iBAAiB,MAAM;CACjE;CAEA,QACE,EAAC,KAAK,WACN,EAAC,YACW;EACZ,MAAM,aACJ,SAAS,OAAO,SAAS,SACrB,SAAS,OAAO,UAChB,KAAKY,WAAW,cAAc,QAAQ,CAAC,EAAE;EAC/C,IAAI,CAAC,YAAY,QACf,MAAM,IAAI,MACR,2BAA2B,SAAS,KAAK,wCAC3C;EAIF,IAAI,YAAY,WAAW,QACzB,OAAO;EAET,MAAM,MAAqC,CAAC;EAC5C,KAAK,MAAM,OAAO,YAChB,IAAI,OAAO,IAAI;EAEjB,OAAO;CACT;CAEA,cAAc,QAAuB;EACnC,MAAM,QAAQ,cAAc,OAAO,QAAQ;EAC3C,MAAM,YAAY,KAAKA,WAAW,KAAK;EACvC,MAAM,SAAS,QAAQ,OAAO,KAAK,WAAW,KAAKH,WAAW;EAE9D,KAAKI,QAAQ,OAAO;GAClB,GAAG,OAAO;IACT,2BAA2B,KAAKL;EACnC,CAAC;EAED,IAAI,OAAO,SAAS,OAAO,QAAQ,WAAW,GAQ5C;EAEF,MAAM,MAAM,KAAKM,QAAQ,QAAQ,MAAM;EACvC,KAAKC,UAAU,OAAO,KAAK,qBAAqB,OAAO,KAAK,SAAS,CAAC;CACxE;CAEA,QAAQ,OAAe,KAAc;EACnC,MAAM,UAAU,OAAO,KAAK,GAAG,EAAE,KAAI,MAAK,GAAG,CAAC,CAAC;EAC/C,KAAKrB,IAAI,IACP;+BACyB,GAAG,KAAK,EAAE,IAAI,QAAQ,KAAK,GAAG,EAAE;kBAC7C,MAAM,KAAK,EAAC,QAAQ,QAAQ,OAAM,CAAC,EAAE,KAAK,GAAG,EAAE,KAAK,GAAG,EAAE;SAErE,OAAO,OAAO,GAAG,CACnB;CACF;CAcA,cAAc,QAAuB;EACnC,MAAM,QAAQ,cAAc,OAAO,QAAQ;EAC3C,MAAM,YAAY,KAAKkB,WAAW,KAAK;EACvC,MAAM,SAAS,QAAQ,OAAO,KAAK,WAAW,KAAKH,WAAW;EAC9D,MAAM,MAAM;GAAC,GAAG,OAAO;IAAM,2BAA2B,KAAKD;EAAQ;EAGrE,MAAM,SAAS,OAAO,MAClB,KAAKM,QACH,QAAQ,OAAO,KAAK,KAAKF,WAAW,KAAK,GAAG,KAAKH,WAAW,GAC5D,MACF,IACA;EACJ,MAAM,SAAS,KAAKK,QAAQ,QAAQ,MAAM;EAE1C,IAAI,QACF,KAAKE,aAAa,OAAO,QAAQ,UAAU,WAAW;EAExD,KAAKD,UAAU,OAAO,QAAQ,qBAAqB,OAAO,KAAK,SAAS,CAAC;EAEzE,MAAM,UAAU,UAAU;EAC1B,MAAM,QAAQ,OAAO,KAAK,OAAO,EAAE,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,GAAG;EAC5D,MAAM,WAAW,OAAO,KAAK,GAAG,EAAE,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,GAAG;EAE3D,IAAI;GACF,MAAM,EAAC,YAAW,KAAKrB,IAAI,IACzB;iBACS,GAAG,KAAK,EAAE;gBACX,SAAS,KAAK,GAAG,EAAE;kBACjB,MAAM,KAAK,OAAO,EAAE;WAE9B,CAAC,GAAG,OAAO,OAAO,GAAG,GAAG,GAAG,OAAO,OAAO,OAAO,CAAC,CACnD;GAIA,IAAI,YAAY,GACd,KAAKmB,QAAQ,OAAO,GAAG;EAE3B,SAAS,GAAG;GACV,IACE,EAAE,aAAa,eACf,EAAE,YAAY,0BAEd,MAAM;GAGR,MAAM,UAAU,CAAC,GAAG,OAAO,QAAQ,GAAG,GAAG,GAAG,OAAO,QAAQ,OAAO,CAAC,EAChE,KACE,CAAC,QAAQ,WACR,CAAC,QAAQ,0BAA0B,KAAK,CAAC,CAC7C,EACC,MAAM,GAAG,iBAAiB,WAAW;GACxC,MAAM,cACJ,iBAAiB,OAAO,YACxB,OAAO,OAAO,SAAS,gBAAgB,WACnC,gBAAgB,OAAO,SAAS,gBAChC;GACN,MAAM,QAAQ,IAAI,MAChB,uCAAuC,KAAKL,WAAW,YAAY,SAAS,OAAO,SAAS,OAAO,GAAG,OAAO,SAAS,KAAK,UAAU,UAAU,MAAM,YAAY,UAAU,KAAK,IAAI,QAAQ,OAAO,MACnM,EAAC,OAAO,EAAC,CACX;GACA,MAAM,OAAO;GACb,MAAM;EACR;CACF;CAEA,cAAc,KAAoB;EAChC,MAAM,QAAQ,cAAc,IAAI,QAAQ;EACxC,MAAM,YAAY,KAAKI,WAAW,KAAK;EACvC,MAAM,SAAS,KAAKE,QAClB,QAAQ,IAAI,KAAK,WAAW,KAAKL,WAAW,GAC5C,GACF;EAEA,KAAKQ,QAAQ,OAAO,MAAM;EAC1B,KAAKD,aAAa,OAAO,QAAQ,UAAU,WAAW;CACxD;CAEA,QAAQ,OAAe,QAAoB;EACzC,MAAM,QAAQ,OAAO,KAAK,MAAM,EAAE,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,GAAG;EAC3D,KAAKtB,IAAI,IACP,eAAe,GAAG,KAAK,EAAE,SAAS,MAAM,KAAK,OAAO,KACpD,OAAO,OAAO,MAAM,CACtB;CACF;CAEA,gBAAgB,UAA2B;EACzC,KAAK,MAAM,YAAY,SAAS,WAAW;GACzC,MAAM,QAAQ,cAAc,QAAQ;GAEpC,KAAKA,IAAI,IAAI,eAAe,GAAG,KAAK,GAAG;GAGvC,KAAKwB,eAAe,KAAK;EAC3B;CACF;CAEA,mBAAmB,QAAqB;EACtC,IAAI,OAAO,UACT,KAAKtB,eAAe,oBAAoB,OAAO,MAAM,OAAO,QAAQ;EAEtE,KAAKC,aAAa,MAAM,MAAM;EAC9B,MAAM,QAAQ,kBAAkB,OAAO,IAAI;EAC3C,KAAKH,IAAI,GAAG,KAAK,yBAAyB,KAAK,CAAC;EAGhD,KAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,OAAO,KAAK,OAAO,GACjE,KAAKgB,gBAAgB,OACnB,MAAM,MACN,SACA,SACA,OAAO,WAAW,QACpB;EAGF,KAAKS,YAAY,MAAM,IAAI;EAC3B,KAAKb,IAAI,OAAO,OAAO,KAAK,MAAM,IAAI;CACxC;CAEA,qBAAqB,KAA0B;EAC7C,KAAKV,eAAe,oBAAoB,IAAI,OAAO,IAAI,GAAG;EAO1D,KAAKC,aAAa,MAAM,GAAG;CAC7B;CAEA,mBAAmB,QAAqB;EACtC,KAAKD,eAAe,OAAO,OAAO,KAAK,OAAO,GAAG;EACjD,KAAKC,aAAa,MAAM,MAAM;EAE9B,MAAM,UAAU,cAAc,OAAO,GAAG;EACxC,MAAM,UAAU,cAAc,OAAO,GAAG;EACxC,KAAKH,IAAI,GAAG,KAAK,eAAe,GAAG,OAAO,EAAE,aAAa,GAAG,OAAO,GAAG;EAGtE,KAAKgB,gBAAgB,YAAY,SAAS,OAAO;EAEjD,KAAKU,cAAc,OAAO,GAAG;EAC7B,KAAKD,YAAY,OAAO;EACxB,KAAKb,IAAI,OAAO,OAAO,KAAK,SAAS,OAAO;CAC9C;CAEA,iBAAiB,KAAgB;EAC/B,IAAI,IAAI,eACN,KAAKV,eAAe,oBAAoB,IAAI,OAAO,IAAI,aAAa;EAEtE,KAAKC,aAAa,MAAM,GAAG;EAC3B,MAAM,QAAQ,cAAc,IAAI,KAAK;EACrC,MAAM,EAAC,SAAQ,IAAI;EACnB,MAAM,OAAO,wBAAwB,OAAO,IAAI,MAAM;EACtD,KAAKH,IAAI,GAAG,KACV,eAAe,GAAG,KAAK,EAAE,OAAO,GAAG,IAAI,EAAE,GAAG,cAAc,IAAI,GAChE;EAGA,KAAKgB,gBAAgB,OAAO,OAAO,MAAM,IAAI,OAAO,MAAM,IAAI,QAAQ;EAEtE,IAAI,IAAI,UACN,KAAKC,kBAAkB;OAIvB,KAAKS,cAAc,IAAI,KAAK;EAE9B,KAAKd,IAAI,OAAO,IAAI,KAAK,OAAO,IAAI,MAAM;CAC5C;CAEA,oBAAoB,KAAmB;EAGrC,KAAKT,aAAa,MAAM,GAAG;EAE3B,MAAM,QAAQ,cAAc,IAAI,KAAK;EACrC,IAAI,UAAU,IAAI,IAAI;EACtB,MAAM,UAAU,IAAI,IAAI;EAExB,MAAM,qBAAqB,sBACzB,IAAI,IAAI,MACR,IAAI,IAAI,IACV;EAMA,IAAI,YAAY,WAAW,CAAC,oBAAoB;GAC9C,KAAKa,gBAAgB,OACnB,OACA,IAAI,IAAI,MACR,IAAI,IAAI,MACR,IAAI,IAAI,IACV;GACA,IAAI,QAAQ,IAAI,IAAI,KAAK,OAAO,MAAM,QAAQ,IAAI,IAAI,KAAK,OAAO,GAChE,KAAKU,cAAc,IAAI,KAAK;GAE9B,KAAKd,IAAI,OAAO,IAAI,KAAK,yBAAyB,IAAI,KAAK,IAAI,GAAG;GAClE;EACF;EAGA,IAAI,oBAAoB;GAWtB,MAAM,cAAc,wBAClB,OACA,IAAI,KACJ,gBACF;GACA,IAAI,YAAY,SAAS;IAIvB,KAAKZ,IAAI,GAAG,KACV,eAAe,GAAG,KAAK,EAAE,UAAU,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,GACjE;IACA,UAAU;GACZ;GACA,MAAM,YAAY,KAChB,WAAW,KAAKA,IAAI,IAAI,OAAO,KAAK,EAAE,MACpC,cAAa,UAAU,SAAS,KAClC,CACF;GAIA,MAAM,WAAW,KAAKA,IAAI,GACvB,QACC;;4BAGF,EACC,IAAmB,KAAK,EACxB,KAAK,EAAC,UAAS,GAAG;GACrB,MAAM,WAAW,OAAO;GACxB,MAAM,UAAU,OAAO,KAAK,UAAU,OAAO;GAC7C,MAAM,QAAQ;IACZ,yBAAyB;KACvB,GAAG;KACH,MAAM;KACN,SAAS,WAAW,UAAU,UAAU,QAAQ,SAAS,CACvD,QACA,WAAW,UAAU;MAAC,GAAG;MAAa,KAAK,KAAK;KAAG,IAAI,IACzD,CAAC;IACH,CAAC;IACD,eAAe,GAAG,QAAQ,EAAE,IAAI,QAAQ,IAAI,EAAE,EAAE,KAAK,GAAG,EAAE;kBAChD,QAAQ,IAAI,EAAE,EAAE,KAAK,GAAG,EAAE,QAAQ,GAAG,KAAK,EAAE;IACtD,cAAc,GAAG,KAAK,EAAE;IACxB,eAAe,GAAG,QAAQ,EAAE,aAAa,GAAG,KAAK,EAAE;IACnD,GAAG,SAAS,KAAI,QAAO,GAAG,IAAI,EAAE;GAClC;GACA,KAAKA,IAAI,GAAG,KAAK,MAAM,KAAK,EAAE,CAAC;EACjC;EACA,IAAI,YAAY,SACd,KAAKA,IAAI,GAAG,KACV,eAAe,GAAG,KAAK,EAAE,UAAU,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,GACjE;EAIF,KAAKgB,gBAAgB,OACnB,OACA,IAAI,IAAI,MACR,IAAI,IAAI,MACR,IAAI,IAAI,IACV;EAEA,KAAKU,cAAc,IAAI,KAAK;EAC5B,KAAKd,IAAI,OAAO,IAAI,KAAK,OAAO,IAAI,GAAG;CACzC;CAEA,kBAAkB,KAAiB;EACjC,MAAM,QAAQ,cAAc,IAAI,KAAK;EACrC,MAAM,EAAC,WAAU;EACjB,KAAKZ,IAAI,GAAG,KAAK,eAAe,GAAG,KAAK,EAAE,QAAQ,GAAG,MAAM,GAAG;EAG9D,KAAKgB,gBAAgB,aAAa,OAAO,MAAM;EAC/C,KAAKb,aAAa,MAAM,GAAG;EAE3B,KAAKuB,cAAc,IAAI,KAAK;EAC5B,KAAKd,IAAI,OAAO,IAAI,KAAK,OAAO,MAAM;CACxC;CAEA,iBAAiB,MAAiB;EAChC,KAAKV,eAAe,KAAK,KAAK,EAAE;EAChC,KAAKC,aAAa,MAAM,IAAI;EAE5B,MAAM,OAAO,cAAc,KAAK,EAAE;EAClC,KAAKH,IAAI,GAAG,KAAK,wBAAwB,GAAG,IAAI,GAAG;EAGnD,KAAKgB,gBAAgB,YAAY,IAAI;EAErC,KAAKS,YAAY,IAAI;EACrB,KAAKb,IAAI,OAAO,KAAK,KAAK,IAAI;CAChC;CAEA,mBAAmB,QAAqB;EACtC,MAAM,QAAQ,uBAAuB,OAAO,IAAI;EAChD,KAAKZ,IAAI,GAAG,KAAK,yBAAyB,KAAK,CAAC;EAIhD,KAAKyB,YAAY,MAAM,SAAS;EAChC,KAAKb,IAAI,OAAO,OAAO,KAAK,MAAM,IAAI;CACxC;CAEA,iBAAiB,MAAiB;EAChC,MAAM,OAAO,cAAc,KAAK,EAAE;EAClC,KAAKZ,IAAI,GAAG,KAAK,wBAAwB,GAAG,IAAI,GAAG;EACnD,KAAKY,IAAI,OAAO,KAAK,KAAK,IAAI;CAChC;CAEA,cAAc,OAAmB;EAC/B,KAAKV,eAAe,iBAAiB,OAAO,KAAKY,QAAQ;EACzD,KAAKW,YAAY,cAAc,KAAK,CAAC;CACvC;;;;CAKA,UACE,OACA,KACA,mBACA;EAIA,IAAI,KAAKrB,UAAU,aAAa,sBAAsB,KAAA,GAAW;GAC/D,KAAKH,WAAW,SACd,KAAKa,UACL,KAAKa,QACL,OACA,KACA,iBACF;GACA,KAAKC;EACP;CACF;CAEA,aAAa,OAAe,KAAiB,aAAwB;EAInE,IAAI,KAAKxB,UAAU,aAAa,aAAa,QAAQ;GACnD,KAAKH,WAAW,YAAY,KAAKa,UAAU,KAAKa,QAAQ,OAAO,GAAG;GAClE,KAAKC;EACP;CACF;CAEA,eAAe,OAAe;EAC5B,IAAI,KAAKxB,UAAU,WAAW;GAC5B,KAAKH,WAAW,cAAc,KAAKa,UAAU,KAAK;GAClD,KAAKc;EACP;CACF;CAEA,YAAY,OAAe;EACzB,KAAKC,iBAAiB;EACtB,IAAI,KAAKzB,UAAU,WAAW;GAC5B,KAAKH,WAAW,WAAW,KAAKa,UAAU,KAAK;GAC/C,KAAKc;EACP;EACA,KAAKX,kBAAkB;CACzB;CAEA,gBAAgB,EAAC,UAAU,WAAW,SAAS,aAA6B;EAC1E,MAAM,YAAY,cAAc,QAAQ;EACxC,MAAM,YAAY,KAAK,KAAKX,YAAY,IAAI,SAAS,CAAC;EACtD,MAAM,aAAa,SAAS,OAAO;EACnC,MAAM,OAAO,CAAC,GAAG,YAAY,GAAG,OAAO;EAQvC,MAAM,iBAAiB,IAAI,IAAI,UAAU,eAAe,CAAC,CAAC;EAC1D,IAAI,CAAC,KAAK,MAAK,MAAK,eAAe,IAAI,CAAC,CAAC,GAAG;GAG1C,KAAKM,IAAI,QACP,kCAAkC,UAAU,OAAO,aAC9C,UAAU,iCACjB;GACA;EACF;EAGA,MAAM,gBAAgB,CAAC,GAAG,MAAM,wBAAwB,EAAE,IAAI,EAAE,EAAE,KAAK,GAAG;EAC1E,MAAM,SAAS,MAAM,KAAK,EAAC,QAAQ,KAAK,SAAS,EAAC,CAAC,EAChD,KAAK,GAAG,EACR,KAAK,GAAG;EACX,MAAM,gBAAgB,WAAW,IAAI,EAAE,EAAE,KAAK,GAAG;EAEjD,IAAI,aAAa;EACjB,IAAI,UAAU;EACd,KAAK,MAAM,KAAK,WAAW;GACzB,MAAM,MAAM,QACV,OAAO,YAAY,KAAK,KAAK,GAAG,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAChD,WACA,KAAKG,WACP;GACA,MAAM,SAAS,KAAKK,QAAQ,KAAK,EAAC,SAAQ,CAAC;GAC3C,MAAM,QAAQ,KAAKnB,WAAW,eAAe,WAAW,MAAM;GAC9D,IAAI,OAAO,OAAA,OAAiB,MAAM,eAAe,WAAW;IAC1D;IACA;GACF;GAGA,MAAM,WACJ,OAAO,OAAA,MACH,KAAK,QACH,OAAM,MAAM,0BAA0B,MAAM,OAAO,SACrD,IACA,MACJ,QAAO,MAAK,eAAe,IAAI,CAAC,CAAC;GACnC,IAAI,QAAQ,WAAW,GAAG;IAGxB;IACA;GACF;GACA,MAAM,cAAc,QAAQ,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,YAAY,GAAG,GAAG,GAAG;GACvE,KAAKD,IAAI,IACC;sBACM,GAAG,SAAS,EAAE,IAAI,cAAc,YAAY,OAAO;yBAChD,cAAc;0BACb,YAAY,KAAK,GAAG,EAAE;SAExC,GAAG,OAAO,OAAO,IAAI,GAAG,GACxB,SACF;GACA;EACF;EAEA,KAAKY,IAAI,QACP,cAAc,WAAW,iBAAiB,QAAQ,SAAS,WAC7D;CACF;CAEA;CAEA,yBAAyB,KAAwB;EAC/C,MAAM,EAAC,UAAU,SAAS,WAAU;EACpC,MAAM,YAAY,cAAc,QAAQ;EAExC,MAAM,OAAO,CAAC,GADK,SAAS,OAAO,SACN,GAAG,OAAO;EAOvC,MAAM,YAAY,KAAK,KAAKN,YAAY,IAAI,SAAS,CAAC;EACtD,MAAM,iBAAiB,IAAI,IAAI,UAAU,eAAe,CAAC,CAAC;EAC1D,IAAI,CAAC,KAAK,MAAK,QAAO,eAAe,IAAI,GAAG,CAAC,GAAG;GAC9C,KAAKM,IAAI,QACP,6CAA6C,UAAU,iCAEzD;GACA;EACF;EAEA,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,KAAKZ,IAAI,EAAE,CAAC;EACxE,KAAK,MAAM,OAAO,MAChB,eAAe,iBAAiB,WAAW,GAAG;EAEhD,KAAKG,aAAa,MAAM,GAAG;EAG3B,KAAKuB,cAAc,QAAQ;EAC3B,IAAI,QACF,KAAKI,qBAAqB;GAAC,OAAO;GAAW,SAAS;GAAM,GAAG;EAAM;EAEvE,KAAKlB,IAAI,OAAO,wBAAwB,WAAW;CAWrD;CAEA,cAAc,QAAuB,WAAiC;EACpE,IAAI,cAAc,KAAKE,UACrB,MAAM,IAAI,MACR,oBAAoB,UAAU,kCAC5B,KAAKA,SACN,IAAI,UAAU,MAAM,GACvB;EAEF,2BAA2B,KAAKd,KAAK,SAAS;EAE9C,IAAI,KAAK6B,gBAAgB;GACvB,MAAM,QAAQ,KAAK,IAAI;GACvB,KAAK7B,IAAI,GAAG,OAAO,UAAU;GAC7B,KAAKY,IAAI,OACP,yCAAyC,KAAK,IAAI,IAAI,MAAM,KAC9D;EACF;EAEA,IAAI,KAAKR,UAAU,gBACjB,KAAKJ,IAAI,OAAO;EAGlB,MAAM,YAAY,KAAK,IAAI,IAAI,KAAKa;EACpC,KAAKD,IAAI,QAAQ,gBAAgB,KAAKE,SAAS,IAAI,UAAU,KAAK;EAElE,OAAO;GACL;GACA,mBAAmB,KAAKgB;GACxB,eAAe,KAAKD;GACpB,kBAAkB,KAAKD,uBAAuB;GAC9C,sBAAsB,OAAO;EAC/B;CACF;CAEA,MAAM,IAAgB;EACpB,GAAG,OAAO,wBAAwB,KAAKd,UAAU;EACjD,KAAKd,IAAI,SAAS;EAIlB,KAAKM,YAAY,MAAM;CACzB;AACF;AAEA,SAAS,sBACP,SACA,SACS;CACT,OACE,QAAQ,aAAa,QAAQ,YAC7B,aAAa,OAAO,MAAM,aAAa,OAAO,KAC9C,cAAc,OAAO,MAAM,cAAc,OAAO;AAEpD;AAEA,SAAS,qBACP,KACA,EAAC,eACqB;CACtB,IAAI,CAAC,aAAa,QAChB;CAEF,OAAO,YAAY,QAAO,QAAO,OAAO,GAAG;AAC7C;AAEA,SAAS,0BAA0B,OAA0C;CAC3E,IAAI,OAAO,UAAU,UACnB,OAAO,OAAO,OAAO,IAAI,KAAK,MAAM,QAChC,qCACA,KAAA;CAGN,MAAM,YACJ,OAAO,UAAU,WACb,OAAO,WAAW,KAAK,IACvB,iBAAiB,aACf,MAAM,aACN,KAAA;CACR,IAAI,cAAc,KAAA,KAAa,YAAY,uBAEzC,OAAO,aADW,OAAO,UAAU,WAAW,WAAW,SAC3B,aAAa,UAAU,cAAc;AAIvE;;AAGA,SAAS,QAAQ,IAAyB;CACxC,IAAI;EACF,GAAG;EACH;CACF,SAAS,GAAG;EACV,OAAO,qBAAK,IAAI,MAAM,iBAAiB;CACzC;AACF;AAEA,SAAS,YAAY,KAAqB;CACxC,IAAI,eAAe,OACjB,OAAO;CAET,MAAM,wBAAQ,IAAI,MAAM;CACxB,MAAM,QAAQ;CACd,OAAO;AACT"}
1	+ {"version":3,"file":"change-processor.js","names":["#db","#changeLog","#tableMetadata","#backfilling","#mode","#failService","#tableSpecs","#failure","#currentTx","#fail","#processMessage","#beginTransaction","#lc","#startMs","#version","#jsonFormat","#columnMetadata","#reloadTableSpecs","#tableSpec","#upsert","#getKey","#logSetOp","#logDeleteOp","#delete","#logTruncateOp","#logResetOp","#bumpVersions","#pos","#numChangeLogEntries","#schemaChanged","#trackBackfillProgress","#completedBackfill"],"sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"sourcesContent":["import {constants as bufferConstants} from 'node:buffer';\nimport type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../shared/src/json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {mapEntries} from '../../../../shared/src/objects.ts';\nimport type {DownloadStatus} from '../../../../zero-events/src/status.ts';\nimport {\n createLiteIndexStatement,\n createLiteTableStatement,\n liteColumnDef,\n} from '../../db/create.ts';\nimport {\n computeZqlSpecs,\n listTables,\n type LiteTableSpecWithReplicationStatus,\n} from '../../db/lite-tables.ts';\nimport {\n isArrayColumn,\n isEnumColumn,\n mapPostgresToLite,\n mapPostgresToLiteColumn,\n mapPostgresToLiteIndex,\n} from '../../db/pg-to-lite.ts';\nimport type {ColumnSpec} from '../../db/specs.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport type {LexiVersion} from '../../types/lexi-version.ts';\nimport {\n JSON_PARSED,\n liteRow,\n type JSONFormat,\n type LiteRow,\n type LiteRowKey,\n type LiteValueType,\n} from '../../types/lite.ts';\nimport {liteTableName} from '../../types/names.ts';\nimport {id} from '../../types/sql.ts';\nimport {\n acceptBackfill,\n type AcceptResult,\n} from '../change-source/protocol/backfill-progress.ts';\nimport type {\n BackfillCompleted,\n BackfillProgressMark,\n Change,\n ColumnAdd,\n ColumnDrop,\n ColumnUpdate,\n Identifier,\n IndexCreate,\n IndexDrop,\n MessageBackfill,\n MessageCommit,\n MessageDelete,\n MessageInsert,\n MessageRelation,\n MessageTruncate,\n MessageUpdate,\n TableCreate,\n TableDrop,\n TableRename,\n TableUpdateMetadata,\n} from '../change-source/protocol/current/data.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from './replicator.ts';\nimport {BackfillingTracker} from './schema/backfilling.ts';\nimport {ChangeLog, DEL_OP, SET_OP} from './schema/change-log.ts';\nimport {ColumnMetadataStore} from './schema/column-metadata.ts';\nimport {\n ZERO_VERSION_COLUMN_NAME,\n updateReplicationWatermark,\n} from './schema/replication-state.ts';\nimport {TableMetadataTracker} from './schema/table-metadata.ts';\n\nexport type ChangeProcessorMode = ReplicatorMode \| 'initial-sync';\n\nconst BIND_VALUE_TOO_BIG_ERROR =\n 'The bound string, buffer, or bigint is too big';\nconst MAX_SQLITE_BIND_BYTES = Math.min(\n bufferConstants.MAX_LENGTH,\n bufferConstants.MAX_STRING_LENGTH,\n);\n\nexport type CommitResult = {\n watermark: string;\n completedBackfill: DownloadStatus \| undefined;\n schemaUpdated: boolean;\n changeLogUpdated: boolean;\n /*\n Millisecond epoch at which the transaction committed upstream, if the\n * ChangeSource reported one. Propagated to ViewSyncers as the origin\n * timestamp of the end-to-end serving lag measurement.\n /\n upstreamCommitTimeMs?: number \| undefined;\n};\n\n/\n The ChangeProcessor partitions the stream of messages into transactions\n * by creating a {@link TransactionProcessor} when a transaction begins, and dispatching\n * messages to it until the commit is received.\n \n From https://www.postgresql.org/docs/current/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW :\n \n \"The logical replication protocol sends individual transactions one by one.\n * This means that all messages between a pair of Begin and Commit messages\n * belong to the same transaction.\"\n /\nexport class ChangeProcessor {\n readonly #db: StatementRunner;\n readonly #changeLog: ChangeLog;\n readonly #tableMetadata: TableMetadataTracker;\n readonly #backfilling: BackfillingTracker;\n readonly #mode: ChangeProcessorMode;\n readonly #failService: (lc: LogContext, err: unknown) => void;\n\n // The TransactionProcessor lazily loads table specs into this Map,\n // and reloads them after a schema change. It is cached here to avoid\n // reading them from the DB on every transaction.\n readonly #tableSpecs = new Map<string, LiteTableSpecWithReplicationStatus>();\n\n #currentTx: TransactionProcessor \| null = null;\n\n #failure: Error \| undefined;\n\n constructor(\n db: StatementRunner,\n mode: ChangeProcessorMode,\n failService: (lc: LogContext, err: unknown) => void,\n ) {\n this.#db = db;\n this.#changeLog = new ChangeLog(db.db);\n this.#tableMetadata = new TableMetadataTracker(db.db);\n this.#backfilling = new BackfillingTracker(db.db);\n this.#mode = mode;\n this.#failService = failService;\n }\n\n #fail(lc: LogContext, err: unknown) {\n if (!this.#failure) {\n let failureError = err;\n const rollbackErrors = [attempt(() => this.#currentTx?.abort(lc))].filter(\n e => e !== undefined,\n );\n\n if (rollbackErrors.length) {\n const combinedError = new Error(\n `Message processing failed and rollback also failed: operation error = ${String(err)}; rollback error = ${rollbackErrors.map(String).join('; ')}`,\n );\n combinedError.cause = err;\n failureError = combinedError;\n }\n\n this.#failure = ensureError(failureError);\n\n if (!(this.#failure instanceof AbortError)) {\n // Propagate the failure up to the service.\n lc.error?.('Message Processing failed:', this.#failure);\n this.#failService(lc, this.#failure);\n }\n }\n }\n\n abort(lc: LogContext) {\n this.#fail(lc, new AbortError());\n }\n\n /* @return If a transaction was committed. /\n processMessage(\n lc: LogContext,\n downstream: ChangeStreamData,\n ): CommitResult \| null {\n const [type, message] = downstream;\n if (this.#failure) {\n lc.debug?.(`Dropping ${message.tag}`);\n return null;\n }\n try {\n const watermark =\n type === 'begin'\n ? downstream[2].commitWatermark\n : type === 'commit'\n ? downstream[2].watermark\n : undefined;\n return this.#processMessage(lc, message, watermark);\n } catch (e) {\n this.#fail(lc, e);\n }\n return null;\n }\n\n #beginTransaction(\n lc: LogContext,\n commitVersion: string,\n jsonFormat: JSONFormat,\n ): TransactionProcessor {\n const start = Date.now();\n\n // litestream can technically hold the lock for an arbitrary amount of time\n // when checkpointing a large commit. Crashing on the busy-timeout in this\n // scenario will either produce a corrupt backup or otherwise prevent\n // replication from proceeding.\n //\n // Instead, retry the lock acquisition indefinitely. If this masks\n // an unknown deadlock situation, manual intervention will be necessary.\n for (let i = 0; ; i++) {\n try {\n return new TransactionProcessor(\n lc,\n this.#db,\n this.#mode,\n this.#changeLog,\n this.#tableMetadata,\n this.#backfilling,\n this.#tableSpecs,\n commitVersion,\n jsonFormat,\n );\n } catch (e) {\n if (e instanceof SqliteError && e.code === 'SQLITE_BUSY') {\n lc.warn?.(\n `SQLITE_BUSY for ${Date.now() - start} ms (attempt ${i + 1}). ` +\n `This is only expected if litestream is performing a large ` +\n `checkpoint.`,\n e,\n );\n continue;\n }\n throw e;\n }\n }\n }\n\n /* @return If a transaction was committed. /\n #processMessage(\n lc: LogContext,\n msg: Change,\n watermark: string \| undefined,\n ): CommitResult \| null {\n if (msg.tag === 'begin') {\n if (this.#currentTx) {\n throw new Error(`Already in a transaction ${stringify(msg)}`);\n }\n this.#currentTx = this.#beginTransaction(\n lc,\n must(watermark),\n msg.json ?? JSON_PARSED,\n );\n return null;\n }\n\n // For non-begin messages, there should be a #currentTx set.\n const tx = this.#currentTx;\n if (!tx) {\n throw new Error(\n `Received message outside of transaction: ${stringify(msg)}`,\n );\n }\n\n if (msg.tag === 'commit') {\n assert(watermark, 'watermark is required for commit messages');\n const result = tx.processCommit(msg, watermark);\n // Clear only after a successful commit so #fail can roll back a commit\n // path that throws before SQLite has committed.\n this.#currentTx = null;\n return result;\n }\n\n if (msg.tag === 'rollback') {\n tx.abort(lc);\n this.#currentTx = null;\n return null;\n }\n\n switch (msg.tag) {\n case 'insert':\n tx.processInsert(msg);\n break;\n case 'update':\n tx.processUpdate(msg);\n break;\n case 'delete':\n tx.processDelete(msg);\n break;\n case 'truncate':\n tx.processTruncate(msg);\n break;\n case 'create-table':\n tx.processCreateTable(msg);\n break;\n case 'rename-table':\n tx.processRenameTable(msg);\n break;\n case 'update-table-metadata':\n tx.processTableMetadata(msg);\n break;\n case 'add-column':\n tx.processAddColumn(msg);\n break;\n case 'update-column':\n tx.processUpdateColumn(msg);\n break;\n case 'drop-column':\n tx.processDropColumn(msg);\n break;\n case 'drop-table':\n tx.processDropTable(msg);\n break;\n case 'create-index':\n tx.processCreateIndex(msg);\n break;\n case 'drop-index':\n tx.processDropIndex(msg);\n break;\n case 'backfill':\n tx.processBackfill(msg);\n break;\n case 'backfill-completed':\n tx.processBackfillCompleted(msg);\n break;\n default:\n unreachable(msg);\n }\n\n return null;\n }\n}\n\n/\n The {@link TransactionProcessor} handles the sequence of messages from\n * upstream, from `BEGIN` to `COMMIT` and executes the corresponding mutations\n * on the {@link postgres.TransactionSql} on the replica.\n \n When applying row contents to the replica, the `_0_version` column is added / updated,\n * and a corresponding entry in the `ChangeLog` is added. The version value is derived\n * from the watermark of the preceding transaction (stored as the `nextStateVersion` in the\n * `ReplicationState` table).\n \n Side note: For non-streaming Postgres transactions, the commitEndLsn (and thus\n * commit watermark) is available in the `begin` message, so it could theoretically\n * be used for the row version of changes within the transaction. However, the\n * commitEndLsn is not available in the streaming (in-progress) transaction\n * protocol, and may not be available for CDC streams of other upstream types.\n * Therefore, the zero replication protocol is designed to not require the commit\n * watermark when a transaction begins.\n \n Also of interest is the fact that all INSERT Messages are logically applied as\n * UPSERTs. See {@link processInsert} for the underlying motivation.\n /\nclass TransactionProcessor {\n readonly #lc: LogContext;\n readonly #startMs: number;\n readonly #db: StatementRunner;\n readonly #mode: ChangeProcessorMode;\n readonly #version: LexiVersion;\n readonly #changeLog: ChangeLog;\n readonly #tableMetadata: TableMetadataTracker;\n readonly #backfilling: BackfillingTracker;\n readonly #tableSpecs: Map<string, LiteTableSpecWithReplicationStatus>;\n readonly #jsonFormat: JSONFormat;\n readonly #columnMetadata: ColumnMetadataStore;\n\n #pos = 0;\n #schemaChanged = false;\n #numChangeLogEntries = 0;\n\n constructor(\n lc: LogContext,\n db: StatementRunner,\n mode: ChangeProcessorMode,\n changeLog: ChangeLog,\n tableMetadata: TableMetadataTracker,\n backfilling: BackfillingTracker,\n tableSpecs: Map<string, LiteTableSpecWithReplicationStatus>,\n commitVersion: LexiVersion,\n jsonFormat: JSONFormat,\n ) {\n this.#startMs = Date.now();\n this.#mode = mode;\n this.#jsonFormat = jsonFormat;\n\n switch (mode) {\n case 'serving':\n // This is the only transaction that commits to the serving replica.\n // Snapshotters use BEGIN CONCURRENT for private changes that are\n // always rolled back, while BEGIN IMMEDIATE lets this writer spill\n // dirty pages during large transactions.\n db.beginImmediate();\n break;\n case 'backup':\n // For the backup-replicator (i.e. replication-manager), there are no View Syncers\n // and thus BEGIN CONCURRENT is not necessary. In fact, BEGIN CONCURRENT can cause\n // deadlocks with forced wal-checkpoints (which `litestream replicate` performs),\n // so it is important to use vanilla transactions in this configuration.\n db.beginImmediate();\n break;\n case 'initial-sync':\n // When the ChangeProcessor is used for initial-sync, the calling code\n // handles the transaction boundaries.\n break;\n default:\n unreachable();\n }\n this.#db = db;\n this.#version = commitVersion;\n this.#lc = lc.withContext('version', commitVersion);\n this.#changeLog = changeLog;\n this.#tableMetadata = tableMetadata;\n this.#backfilling = backfilling;\n this.#tableSpecs = tableSpecs;\n // The column_metadata table is guaranteed to exist since the\n // replica-schema.ts migration to v8.\n this.#columnMetadata = must(ColumnMetadataStore.getInstance(db.db));\n\n if (this.#tableSpecs.size === 0) {\n this.#reloadTableSpecs();\n }\n }\n\n #reloadTableSpecs() {\n this.#tableSpecs.clear();\n // zqlSpecs include the primary key derived from unique indexes\n const zqlSpecs = computeZqlSpecs(this.#lc, this.#db.db, {\n includeBackfillingColumns: true,\n });\n for (let spec of listTables(this.#db.db)) {\n if (!spec.primaryKey) {\n spec = {\n ...spec,\n primaryKey: [\n ...(zqlSpecs.get(spec.name)?.tableSpec.primaryKey ?? []),\n ],\n };\n }\n this.#tableSpecs.set(spec.name, spec);\n }\n }\n\n #tableSpec(name: string) {\n return must(this.#tableSpecs.get(name), `Unknown table ${name}`);\n }\n\n #getKey(\n {row, numCols}: {row: LiteRow; numCols: number},\n {relation}: {relation: MessageRelation},\n ): LiteRowKey {\n const keyColumns =\n relation.rowKey.type !== 'full'\n ? relation.rowKey.columns // already a suitable key\n : this.#tableSpec(liteTableName(relation)).primaryKey;\n if (!keyColumns?.length) {\n throw new Error(\n `Cannot replicate table \"${relation.name}\" without a PRIMARY KEY or UNIQUE INDEX`,\n );\n }\n // For the common case (replica identity default), the row is already the\n // key for deletes and updates, in which case a new object can be avoided.\n if (numCols === keyColumns.length) {\n return row;\n }\n const key: Record<string, LiteValueType> = {};\n for (const col of keyColumns) {\n key[col] = row[col];\n }\n return key;\n }\n\n processInsert(insert: MessageInsert) {\n const table = liteTableName(insert.relation);\n const tableSpec = this.#tableSpec(table);\n const newRow = liteRow(insert.new, tableSpec, this.#jsonFormat);\n\n this.#upsert(table, {\n ...newRow.row,\n [ZERO_VERSION_COLUMN_NAME]: this.#version,\n });\n\n if (insert.relation.rowKey.columns.length === 0) {\n // INSERTs can be replicated for rows without a PRIMARY KEY or a\n // UNIQUE INDEX. These are written to the replica but not recorded\n // in the changeLog, because these rows cannot participate in IVM.\n //\n // (Once the table schema has been corrected to include a key, the\n // associated schema change will reset pipelines and data can be\n // loaded via hydration.)\n return;\n }\n const key = this.#getKey(newRow, insert);\n this.#logSetOp(table, key, getBackfilledColumns(newRow.row, tableSpec));\n }\n\n #upsert(table: string, row: LiteRow) {\n const columns = Object.keys(row).map(c => id(c));\n this.#db.run(\n `\n INSERT OR REPLACE INTO ${id(table)} (${columns.join(',')})\n VALUES (${Array.from({length: columns.length}).fill('?').join(',')})\n `,\n Object.values(row),\n );\n }\n\n // Updates by default are applied as UPDATE commands to support partial\n // row specifications from the change source. In particular, this is needed\n // to handle updates for which unchanged TOASTed values are not sent:\n //\n // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html#PROTOCOL-LOGICALREP-MESSAGE-FORMATS-TUPLEDATA\n //\n // However, in certain cases an UPDATE may be received for a row that\n // was not initially synced, such as when, an existing table is added\n // to the app's publication.\n //\n // In order to facilitate \"resumptive\" replication, the logic falls back to\n // an INSERT if the update did not change any rows.\n processUpdate(update: MessageUpdate) {\n const table = liteTableName(update.relation);\n const tableSpec = this.#tableSpec(table);\n const newRow = liteRow(update.new, tableSpec, this.#jsonFormat);\n const row = {...newRow.row, [ZERO_VERSION_COLUMN_NAME]: this.#version};\n\n // update.key is set with the old values if the key has changed.\n const oldKey = update.key\n ? this.#getKey(\n liteRow(update.key, this.#tableSpec(table), this.#jsonFormat),\n update,\n )\n : null;\n const newKey = this.#getKey(newRow, update);\n\n if (oldKey) {\n this.#logDeleteOp(table, oldKey, tableSpec.backfilling);\n }\n this.#logSetOp(table, newKey, getBackfilledColumns(newRow.row, tableSpec));\n\n const currKey = oldKey ?? newKey;\n const conds = Object.keys(currKey).map(col => `${id(col)}=?`);\n const setExprs = Object.keys(row).map(col => `${id(col)}=?`);\n\n try {\n const {changes} = this.#db.run(\n `\n UPDATE ${id(table)}\n SET ${setExprs.join(',')}\n WHERE ${conds.join(' AND ')}\n `,\n [...Object.values(row), ...Object.values(currKey)],\n );\n\n // If the UPDATE did not affect any rows, perform an UPSERT of the\n // new row for resumptive replication.\n if (changes === 0) {\n this.#upsert(table, row);\n }\n } catch (e) {\n if (\n !(e instanceof RangeError) \|\|\n e.message !== BIND_VALUE_TOO_BIG_ERROR\n ) {\n throw e;\n }\n\n const binding = [...Object.entries(row), ...Object.entries(currKey)]\n .map(\n ([column, value]) =>\n [column, oversizedValueDescription(value)] as const,\n )\n .find(([, description]) => description);\n const relationOid =\n 'relationOid' in update.relation &&\n typeof update.relation.relationOid === 'number'\n ? ` relationOid=${update.relation.relationOid}`\n : '';\n const error = new Error(\n `Oversized SQLite update binding: tx=${this.#version}${relationOid} table=${update.relation.schema}.${update.relation.name} column=${binding?.[0] ?? 'unknown'}${binding?.[1] ? ` ${binding[1]}` : ''}`,\n {cause: e},\n );\n error.name = 'OversizedUpdateBindingError';\n throw error;\n }\n }\n\n processDelete(del: MessageDelete) {\n const table = liteTableName(del.relation);\n const tableSpec = this.#tableSpec(table);\n const rowKey = this.#getKey(\n liteRow(del.key, tableSpec, this.#jsonFormat),\n del,\n );\n\n this.#delete(table, rowKey);\n this.#logDeleteOp(table, rowKey, tableSpec.backfilling);\n }\n\n #delete(table: string, rowKey: LiteRowKey) {\n const conds = Object.keys(rowKey).map(col => `${id(col)}=?`);\n this.#db.run(\n `DELETE FROM ${id(table)} WHERE ${conds.join(' AND ')}`,\n Object.values(rowKey),\n );\n }\n\n processTruncate(truncate: MessageTruncate) {\n for (const relation of truncate.relations) {\n const table = liteTableName(relation);\n // Update replica data.\n this.#db.run(`DELETE FROM ${id(table)}`);\n\n // Update change log.\n this.#logTruncateOp(table);\n }\n }\n\n processCreateTable(create: TableCreate) {\n if (create.metadata) {\n this.#tableMetadata.setUpstreamMetadata(create.spec, create.metadata);\n }\n this.#backfilling.apply(create);\n const table = mapPostgresToLite(create.spec);\n this.#db.db.exec(createLiteTableStatement(table));\n\n // Write to metadata table\n for (const [colName, colSpec] of Object.entries(create.spec.columns)) {\n this.#columnMetadata.insert(\n table.name,\n colName,\n colSpec,\n create.backfill?.[colName],\n );\n }\n\n this.#logResetOp(table.name);\n this.#lc.info?.(create.tag, table.name);\n }\n\n processTableMetadata(msg: TableUpdateMetadata) {\n this.#tableMetadata.setUpstreamMetadata(msg.table, msg.new);\n // Inert today: the fold's only op for this tag is `upsert-metadata`, which\n // the line above is the replica's interpreter of. Called anyway so that\n // every site that hands a schema change to one cookie tracker hands it to\n // both -- an exception here would be a schema change the shared fold has an\n // opinion about and this store never sees, which is the drift the fold\n // exists to make impossible.\n this.#backfilling.apply(msg);\n }\n\n processRenameTable(rename: TableRename) {\n this.#tableMetadata.rename(rename.old, rename.new);\n this.#backfilling.apply(rename);\n\n const oldName = liteTableName(rename.old);\n const newName = liteTableName(rename.new);\n this.#db.db.exec(`ALTER TABLE ${id(oldName)} RENAME TO ${id(newName)}`);\n\n // Rename in metadata table\n this.#columnMetadata.renameTable(oldName, newName);\n\n this.#bumpVersions(rename.new);\n this.#logResetOp(oldName);\n this.#lc.info?.(rename.tag, oldName, newName);\n }\n\n processAddColumn(msg: ColumnAdd) {\n if (msg.tableMetadata) {\n this.#tableMetadata.setUpstreamMetadata(msg.table, msg.tableMetadata);\n }\n this.#backfilling.apply(msg);\n const table = liteTableName(msg.table);\n const {name} = msg.column;\n const spec = mapPostgresToLiteColumn(table, msg.column);\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} ADD ${id(name)} ${liteColumnDef(spec)}`,\n );\n\n // Write to metadata table\n this.#columnMetadata.insert(table, name, msg.column.spec, msg.backfill);\n\n if (msg.backfill) {\n this.#reloadTableSpecs();\n } else {\n // Make the new column visible immediately if it's not being backfilled.\n // Otherwise, the version bump will happen with the backfill is complete.\n this.#bumpVersions(msg.table);\n }\n this.#lc.info?.(msg.tag, table, msg.column);\n }\n\n processUpdateColumn(msg: ColumnUpdate) {\n // A no-op unless this is a rename, which is the only part of a column\n // update that moves the backfill cookie.\n this.#backfilling.apply(msg);\n\n const table = liteTableName(msg.table);\n let oldName = msg.old.name;\n const newName = msg.new.name;\n\n const storageTypesDiffer = differentStorageTypes(\n msg.old.spec,\n msg.new.spec,\n );\n\n // If neither the column name nor the SQLite data type changes, only the\n // upstream metadata needs to be updated. This includes changes such as a\n // varchar character limit and nullability, which SQLite does not enforce\n // but a freshly built replica still records.\n if (oldName === newName && !storageTypesDiffer) {\n this.#columnMetadata.update(\n table,\n msg.old.name,\n msg.new.name,\n msg.new.spec,\n );\n if (Boolean(msg.old.spec.notNull) !== Boolean(msg.new.spec.notNull)) {\n this.#bumpVersions(msg.table);\n }\n this.#lc.info?.(msg.tag, 'updated metadata only', msg.old, msg.new);\n return;\n }\n // Storage type changes require a new column so that SQLite uses the new\n // representation for existing and future values.\n if (storageTypesDiffer) {\n // update-column can ignore defaults because it does not change the values\n // in existing rows.\n //\n // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DESC-SET-DROP-DEFAULT\n //\n // \"The new default value will only apply in subsequent INSERT or UPDATE\n // commands; it does not cause rows already in the table to change.\"\n //\n // This allows support for _changing_ column defaults to any expression,\n // since it does not affect what the replica needs to do.\n const newLiteSpec = mapPostgresToLiteColumn(\n table,\n msg.new,\n 'ignore-default',\n );\n if (oldName !== newName) {\n // Rename the column in place first so that SQLite rewrites every\n // reference to it in the table's index definitions, including the\n // WHERE clauses of partial indexes.\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} RENAME ${id(oldName)} TO ${id(newName)}`,\n );\n oldName = newName;\n }\n const tableSpec = must(\n listTables(this.#db.db, false, false).find(\n tableSpec => tableSpec.name === table,\n ),\n );\n // Recreating the table drops its indexes, so capture their DDL to\n // replay afterwards. Auto-indexes (PRIMARY KEY) have no DDL and are\n // recreated by the CREATE TABLE statement itself.\n const indexDDL = this.#db.db\n .prepare(\n `SELECT sql FROM sqlite_master\n WHERE type = 'index' AND tbl_name = ? AND sql IS NOT NULL\n ORDER BY rowid`,\n )\n .all<{sql: string}>(table)\n .map(({sql}) => sql);\n const tmpTable = `tmp.${table}`;\n const columns = Object.keys(tableSpec.columns);\n const stmts = [\n createLiteTableStatement({\n ...tableSpec,\n name: tmpTable,\n columns: mapEntries(tableSpec.columns, (column, spec) => [\n column,\n column === newName ? {...newLiteSpec, pos: spec.pos} : spec,\n ]),\n }),\n `INSERT INTO ${id(tmpTable)} (${columns.map(id).join(',')})\n SELECT ${columns.map(id).join(',')} FROM ${id(table)};`,\n `DROP TABLE ${id(table)};`,\n `ALTER TABLE ${id(tmpTable)} RENAME TO ${id(table)};`,\n ...indexDDL.map(sql => `${sql};`),\n ];\n this.#db.db.exec(stmts.join(''));\n }\n if (oldName !== newName) {\n this.#db.db.exec(\n `ALTER TABLE ${id(table)} RENAME ${id(oldName)} TO ${id(newName)}`,\n );\n }\n\n // Update metadata table\n this.#columnMetadata.update(\n table,\n msg.old.name,\n msg.new.name,\n msg.new.spec,\n );\n\n this.#bumpVersions(msg.table);\n this.#lc.info?.(msg.tag, table, msg.new);\n }\n\n processDropColumn(msg: ColumnDrop) {\n const table = liteTableName(msg.table);\n const {column} = msg;\n this.#db.db.exec(`ALTER TABLE ${id(table)} DROP ${id(column)}`);\n\n // Delete from metadata table\n this.#columnMetadata.deleteColumn(table, column);\n this.#backfilling.apply(msg);\n\n this.#bumpVersions(msg.table);\n this.#lc.info?.(msg.tag, table, column);\n }\n\n processDropTable(drop: TableDrop) {\n this.#tableMetadata.drop(drop.id);\n this.#backfilling.apply(drop);\n\n const name = liteTableName(drop.id);\n this.#db.db.exec(`DROP TABLE IF EXISTS ${id(name)}`);\n\n // Delete from metadata table\n this.#columnMetadata.deleteTable(name);\n\n this.#logResetOp(name);\n this.#lc.info?.(drop.tag, name);\n }\n\n processCreateIndex(create: IndexCreate) {\n const index = mapPostgresToLiteIndex(create.spec);\n this.#db.db.exec(createLiteIndexStatement(index));\n\n // indexes affect tables visibility (e.g. sync-ability is gated on\n // having a unique index), so reset pipelines to refresh table schemas.\n this.#logResetOp(index.tableName);\n this.#lc.info?.(create.tag, index.name);\n }\n\n processDropIndex(drop: IndexDrop) {\n const name = liteTableName(drop.id);\n this.#db.db.exec(`DROP INDEX IF EXISTS ${id(name)}`);\n this.#lc.info?.(drop.tag, name);\n }\n\n #bumpVersions(table: Identifier) {\n this.#tableMetadata.setMinRowVersion(table, this.#version);\n this.#logResetOp(liteTableName(table));\n }\n\n /\n @param backfilledColumns `backfilling` columns for which values were set\n /\n #logSetOp(\n table: string,\n key: LiteRowKey,\n backfilledColumns: string[] \| undefined,\n ) {\n // The \"serving\" replicator always writes to the change-log (for IVM).\n // The \"backup\" replicator only needs to write to the change log\n // when writing columns that are being backfilled.\n if (this.#mode === 'serving' \|\| backfilledColumns !== undefined) {\n this.#changeLog.logSetOp(\n this.#version,\n this.#pos++,\n table,\n key,\n backfilledColumns,\n );\n this.#numChangeLogEntries++;\n }\n }\n\n #logDeleteOp(table: string, key: LiteRowKey, backfilling?: string[]) {\n // The \"serving\" replicator always writes to the change-log (for IVM).\n // The \"backup\" replicator only needs to write to the change log\n // when writing columns that are being backfilled.\n if (this.#mode === 'serving' \|\| backfilling?.length) {\n this.#changeLog.logDeleteOp(this.#version, this.#pos++, table, key);\n this.#numChangeLogEntries++;\n }\n }\n\n #logTruncateOp(table: string) {\n if (this.#mode === 'serving') {\n this.#changeLog.logTruncateOp(this.#version, table);\n this.#numChangeLogEntries++;\n }\n }\n\n #logResetOp(table: string) {\n this.#schemaChanged = true;\n if (this.#mode === 'serving') {\n this.#changeLog.logResetOp(this.#version, table);\n this.#numChangeLogEntries++;\n }\n this.#reloadTableSpecs();\n }\n\n processBackfill({\n relation,\n watermark,\n columns,\n rowValues,\n progressMarks,\n }: MessageBackfill) {\n const tableName = liteTableName(relation);\n const tableSpec = must(this.#tableSpecs.get(tableName));\n const rowKeyCols = relation.rowKey.columns;\n const cols = [...rowKeyCols, ...columns];\n\n // Only columns that are still being backfilled on _this_ replica are\n // written, and of those, only the columns for which this message is\n // contiguous with the data received so far (whose progress is advanced).\n //\n // Once a column has been published (backfill completed), its values\n // strictly follow the replication timeline, so a subsequent backfill must\n // never overwrite them. This guards against a redundant backfill, e.g.\n // when the view-syncer (re)connects to a replication-manager that is\n // (re)running a backfill this replica already completed.\n // * Data for non-contiguous (ignored) columns is necessarily re-delivered\n // (by a resumed or from-scratch run) before the column can be published,\n // so writing it would be wasted I/O.\n const {accepted, ignored} = this.#trackBackfillProgress(\n relation,\n cols,\n own => acceptBackfill(own, progressMarks),\n );\n if (accepted.length === 0) {\n this.#lc.debug?.(\n ignored.length\n ? `skipping backfill of ${rowValues.length} rows into ` +\n `${tableName}: not contiguous with the progress of columns ` +\n `[${ignored.join(',')}]`\n : `skipping redundant backfill of ${rowValues.length} rows into ` +\n `${tableName} (all columns already published)`,\n progressMarks,\n );\n return;\n }\n if (ignored.length) {\n this.#lc.debug?.(\n `backfill of ${tableName} is not contiguous with the progress ` +\n `of columns [${ignored.join(',')}]`,\n progressMarks,\n );\n }\n const acceptedSet = new Set(accepted);\n\n // Common parts of the INSERT sql statement.\n const insertColsStr = [...cols, ZERO_VERSION_COLUMN_NAME].map(id).join(',');\n const qMarks = Array.from({length: cols.length + 1})\n .fill('?')\n .join(',');\n const rowKeyColsStr = rowKeyCols.map(id).join(',');\n\n let backfilled = 0;\n let skipped = 0;\n for (const v of rowValues) {\n const row = liteRow(\n Object.fromEntries(cols.map((c, i) => [c, v[i]])),\n tableSpec,\n this.#jsonFormat,\n );\n const rowKey = this.#getKey(row, {relation});\n const rowOp = this.#changeLog.getLatestRowOp(tableName, rowKey);\n if (rowOp?.op === DEL_OP && rowOp.stateVersion > watermark) {\n skipped++;\n continue; // the row was deleted after the backfill snapshot\n }\n // Restrict the DO UPDATE columns to those still being backfilled. Never\n // update an already backfilled (i.e. published) column.\n const updates = (\n rowOp?.op === SET_OP\n ? cols.filter(\n c => (rowOp.backfillingColumnVersions[c] ?? '') <= watermark,\n )\n : cols\n ).filter(c => acceptedSet.has(c));\n if (updates.length === 0) {\n // row already has newer values for all accepted backfilling columns.\n skipped++;\n continue;\n }\n const updateStmts = updates.map(col => `${id(col)}=excluded.${id(col)}`);\n this.#db.run(\n /sql/ `\n INSERT INTO ${id(tableName)} (${insertColsStr}) VALUES (${qMarks})\n ON CONFLICT (${rowKeyColsStr})\n DO UPDATE SET ${updateStmts.join(',')};\n `,\n ...Object.values(row.row),\n watermark, // the _0_version for new rows (i.e. table backfill)\n );\n backfilled++;\n }\n\n this.#lc.debug?.(\n `backfilled ${backfilled} rows (skipped ${skipped}) into ${tableName}`,\n );\n }\n\n #completedBackfill: DownloadStatus \| undefined;\n\n /*\n Applies `accept` to the tracked progress of each of the `cols` that are\n * still backfilling on this replica, updating the progress of the accepted\n * ones. Columns that are not backfilling (e.g. already published) are\n * neither accepted nor ignored.\n /\n #trackBackfillProgress(\n relation: Identifier,\n cols: readonly string[],\n accept: (progress: BackfillProgressMark \| undefined) => AcceptResult,\n ): {accepted: string[]; ignored: string[]} {\n const tracked = this.#backfilling.getProgress(relation);\n const accepted: string[] = [];\n const ignored: string[] = [];\n for (const col of cols) {\n if (!tracked.has(col)) {\n continue; // not backfilling\n }\n const progress = tracked.get(col);\n const result = accept(progress);\n if (!result.accept) {\n ignored.push(col);\n continue;\n }\n accepted.push(col);\n if (!deepEqual(result.progress, progress)) {\n this.#backfilling.setProgress(relation, col, result.progress);\n }\n }\n return {accepted, ignored};\n }\n\n processBackfillCompleted(msg: BackfillCompleted) {\n const {relation, columns, status, progressMarks} = msg;\n const tableName = liteTableName(relation);\n const rowKeyCols = relation.rowKey.columns;\n\n // Only complete the columns whose backfill progress is contiguous with\n // the completed backfill. The others remain backfilling (and will be\n // re-requested by the change-streamer).\n const {accepted, ignored} = this.#trackBackfillProgress(\n relation,\n [...rowKeyCols, ...columns],\n own =>\n acceptBackfill(\n own,\n progressMarks ? {previous: progressMarks.previous} : undefined,\n ),\n );\n if (ignored.length) {\n this.#lc.info?.(\n `ignoring backfill-completed of ${tableName} for columns ` +\n `[${ignored.join(',')}], which have not received all of the data`,\n progressMarks,\n );\n }\n\n // If none of the columns are still backfilling on this replica, they were\n // already published (e.g. a redundant backfill from a replication-manager\n // re-running a backfill this replica already completed). Bumping versions\n // again would spuriously reset pipelines and re-report completion, so treat\n // the completion as a no-op.\n if (accepted.length === 0) {\n if (ignored.length === 0) {\n this.#lc.debug?.(\n `skipping redundant backfill-completed for ${tableName} ` +\n `(all columns already published)`,\n );\n }\n return;\n }\n const ignoredSet = new Set(ignored);\n const cols = [\n ...rowKeyCols.filter(c => !ignoredSet.has(c)),\n ...columns.filter(c => !ignoredSet.has(c)),\n ];\n\n const columnMetadata = must(ColumnMetadataStore.getInstance(this.#db.db));\n for (const col of cols) {\n columnMetadata.clearBackfilling(tableName, col);\n }\n this.#backfilling.complete(relation, cols);\n // Given that new columns are being exposed for every row in the table, bump the\n // row version for all rows.\n this.#bumpVersions(relation);\n if (status) {\n this.#completedBackfill = {table: tableName, columns: cols, ...status};\n }\n this.#lc.info?.(`finished backfilling ${tableName}`);\n\n // Note that there is no need to clear the backfillingColumnVersions values\n // in the changeLog. It could theoretically be done for clarity but:\n // (1) it could be non-trivial in terms of latency introduced and\n // (2) the data must be preserved if _other_ columns are in the process\n // of being backfilled\n //\n // Thus, for speed and simplicity, the values are left as is. (Note that\n // subsequent replicated changes to those rows will clear the values if\n // no backfills are in progress).\n }\n\n processCommit(commit: MessageCommit, watermark: string): CommitResult {\n if (watermark !== this.#version) {\n throw new Error(\n `'commit' version ${watermark} does not match 'begin' version ${\n this.#version\n }: ${stringify(commit)}`,\n );\n }\n updateReplicationWatermark(this.#db, watermark);\n\n if (this.#schemaChanged) {\n const start = Date.now();\n this.#db.db.pragma('optimize');\n this.#lc.info?.(\n `PRAGMA optimized after schema change (${Date.now() - start} ms)`,\n );\n }\n\n if (this.#mode !== 'initial-sync') {\n this.#db.commit();\n }\n\n const elapsedMs = Date.now() - this.#startMs;\n this.#lc.debug?.(`Committed tx@${this.#version} (${elapsedMs} ms)`);\n\n return {\n watermark,\n completedBackfill: this.#completedBackfill,\n schemaUpdated: this.#schemaChanged,\n changeLogUpdated: this.#numChangeLogEntries > 0,\n upstreamCommitTimeMs: commit.commitTimeMs,\n };\n }\n\n abort(lc: LogContext) {\n lc.info?.(`aborting transaction ${this.#version}`);\n this.#db.rollback();\n // A schema change in the transaction reloaded the table specs, which now\n // describe the schema that the rollback undid. Clearing them makes the\n // next transaction reload them from the database.\n this.#tableSpecs.clear();\n }\n}\n\nfunction differentStorageTypes(\n oldSpec: ColumnSpec,\n newSpec: ColumnSpec,\n): boolean {\n return (\n oldSpec.dataType !== newSpec.dataType \|\|\n isEnumColumn(oldSpec) !== isEnumColumn(newSpec) \|\|\n isArrayColumn(oldSpec) !== isArrayColumn(newSpec)\n );\n}\n\nfunction getBackfilledColumns(\n row: LiteRow,\n {backfilling}: LiteTableSpecWithReplicationStatus,\n): string[] \| undefined {\n if (!backfilling?.length) {\n return undefined; // common case\n }\n return backfilling.filter(col => col in row);\n}\n\nfunction oversizedValueDescription(value: LiteValueType): string \| undefined {\n if (typeof value === 'bigint') {\n return BigInt.asIntN(64, value) !== value\n ? 'valueType=bigint fitsInt64=false'\n : undefined;\n }\n\n const sizeBytes =\n typeof value === 'string'\n ? Buffer.byteLength(value)\n : value instanceof Uint8Array\n ? value.byteLength\n : undefined;\n if (sizeBytes !== undefined && sizeBytes > MAX_SQLITE_BIND_BYTES) {\n const valueType = typeof value === 'string' ? 'string' : 'buffer';\n return `valueType=${valueType} sizeBytes=${sizeBytes} limitBytes=${MAX_SQLITE_BIND_BYTES}`;\n }\n\n return undefined;\n}\n\n/* Runs `fn`, returning its error rather than throwing it. */\nfunction attempt(fn: () => void): unknown {\n try {\n fn();\n return undefined;\n } catch (e) {\n return e ?? new Error('rollback failed');\n }\n}\n\nfunction ensureError(err: unknown): Error {\n if (err instanceof Error) {\n return err;\n }\n const error = new Error();\n error.cause = err;\n return error;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA+EA,IAAM,2BACJ;AACF,IAAM,wBAAwB,KAAK,IACjC,UAAgB,YAChB,UAAgB,iBAClB;;;;;;;;;;;;AA0BA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CACA;CACA;CAKA,8BAAuB,IAAI,IAAgD;CAE3E,aAA0C;CAE1C;CAEA,YACE,IACA,MACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,aAAa,IAAI,UAAU,GAAG,EAAE;EACrC,KAAKC,iBAAiB,IAAI,qBAAqB,GAAG,EAAE;EACpD,KAAKC,eAAe,IAAI,mBAAmB,GAAG,EAAE;EAChD,KAAKC,QAAQ;EACb,KAAKC,eAAe;CACtB;CAEA,MAAM,IAAgB,KAAc;EAClC,IAAI,CAAC,KAAKE,UAAU;GAClB,IAAI,eAAe;GACnB,MAAM,iBAAiB,CAAC,cAAc,KAAKC,YAAY,MAAM,EAAE,CAAC,CAAC,EAAE,QACjE,MAAK,MAAM,KAAA,CACb;GAEA,IAAI,eAAe,QAAQ;IACzB,MAAM,gCAAgB,IAAI,MACxB,yEAAyE,OAAO,GAAG,EAAE,qBAAqB,eAAe,IAAI,MAAM,EAAE,KAAK,IAAI,GAChJ;IACA,cAAc,QAAQ;IACtB,eAAe;GACjB;GAEA,KAAKD,WAAW,YAAY,YAAY;GAExC,IAAI,EAAE,KAAKA,oBAAoB,aAAa;IAE1C,GAAG,QAAQ,8BAA8B,KAAKA,QAAQ;IACtD,KAAKF,aAAa,IAAI,KAAKE,QAAQ;GACrC;EACF;CACF;CAEA,MAAM,IAAgB;EACpB,KAAKE,MAAM,IAAI,IAAI,WAAW,CAAC;CACjC;;CAGA,eACE,IACA,YACqB;EACrB,MAAM,CAAC,MAAM,WAAW;EACxB,IAAI,KAAKF,UAAU;GACjB,GAAG,QAAQ,YAAY,QAAQ,KAAK;GACpC,OAAO;EACT;EACA,IAAI;GACF,MAAM,YACJ,SAAS,UACL,WAAW,GAAG,kBACd,SAAS,WACP,WAAW,GAAG,YACd,KAAA;GACR,OAAO,KAAKG,gBAAgB,IAAI,SAAS,SAAS;EACpD,SAAS,GAAG;GACV,KAAKD,MAAM,IAAI,CAAC;EAClB;EACA,OAAO;CACT;CAEA,kBACE,IACA,eACA,YACsB;EACtB,MAAM,QAAQ,KAAK,IAAI;EASvB,KAAK,IAAI,IAAI,IAAK,KAChB,IAAI;GACF,OAAO,IAAI,qBACT,IACA,KAAKT,KACL,KAAKI,OACL,KAAKH,YACL,KAAKC,gBACL,KAAKC,cACL,KAAKG,aACL,eACA,UACF;EACF,SAAS,GAAG;GACV,IAAI,aAAa,eAAe,EAAE,SAAS,eAAe;IACxD,GAAG,OACD,mBAAmB,KAAK,IAAI,IAAI,MAAM,eAAe,IAAI,EAAE,2EAG3D,CACF;IACA;GACF;GACA,MAAM;EACR;CAEJ;;CAGA,gBACE,IACA,KACA,WACqB;EACrB,IAAI,IAAI,QAAQ,SAAS;GACvB,IAAI,KAAKE,YACP,MAAM,IAAI,MAAM,4BAA4B,UAAU,GAAG,GAAG;GAE9D,KAAKA,aAAa,KAAKG,kBACrB,IACA,KAAK,SAAS,GACd,IAAI,QAAA,GACN;GACA,OAAO;EACT;EAGA,MAAM,KAAK,KAAKH;EAChB,IAAI,CAAC,IACH,MAAM,IAAI,MACR,4CAA4C,UAAU,GAAG,GAC3D;EAGF,IAAI,IAAI,QAAQ,UAAU;GACxB,OAAO,WAAW,2CAA2C;GAC7D,MAAM,SAAS,GAAG,cAAc,KAAK,SAAS;GAG9C,KAAKA,aAAa;GAClB,OAAO;EACT;EAEA,IAAI,IAAI,QAAQ,YAAY;GAC1B,GAAG,MAAM,EAAE;GACX,KAAKA,aAAa;GAClB,OAAO;EACT;EAEA,QAAQ,IAAI,KAAZ;GACE,KAAK;IACH,GAAG,cAAc,GAAG;IACpB;GACF,KAAK;IACH,GAAG,cAAc,GAAG;IACpB;GACF,KAAK;IACH,GAAG,cAAc,GAAG;IACpB;GACF,KAAK;IACH,GAAG,gBAAgB,GAAG;IACtB;GACF,KAAK;IACH,GAAG,mBAAmB,GAAG;IACzB;GACF,KAAK;IACH,GAAG,mBAAmB,GAAG;IACzB;GACF,KAAK;IACH,GAAG,qBAAqB,GAAG;IAC3B;GACF,KAAK;IACH,GAAG,iBAAiB,GAAG;IACvB;GACF,KAAK;IACH,GAAG,oBAAoB,GAAG;IAC1B;GACF,KAAK;IACH,GAAG,kBAAkB,GAAG;IACxB;GACF,KAAK;IACH,GAAG,iBAAiB,GAAG;IACvB;GACF,KAAK;IACH,GAAG,mBAAmB,GAAG;IACzB;GACF,KAAK;IACH,GAAG,iBAAiB,GAAG;IACvB;GACF,KAAK;IACH,GAAG,gBAAgB,GAAG;IACtB;GACF,KAAK;IACH,GAAG,yBAAyB,GAAG;IAC/B;GACF,SACE,YAAY,GAAG;EACnB;EAEA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,OAAO;CACP,iBAAiB;CACjB,uBAAuB;CAEvB,YACE,IACA,IACA,MACA,WACA,eACA,aACA,YACA,eACA,YACA;EACA,KAAKK,WAAW,KAAK,IAAI;EACzB,KAAKT,QAAQ;EACb,KAAKW,cAAc;EAEnB,QAAQ,MAAR;GACE,KAAK;IAKH,GAAG,eAAe;IAClB;GACF,KAAK;IAKH,GAAG,eAAe;IAClB;GACF,KAAK,gBAGH;GACF,SACE,YAAY;EAChB;EACA,KAAKf,MAAM;EACX,KAAKc,WAAW;EAChB,KAAKF,MAAM,GAAG,YAAY,WAAW,aAAa;EAClD,KAAKX,aAAa;EAClB,KAAKC,iBAAiB;EACtB,KAAKC,eAAe;EACpB,KAAKG,cAAc;EAGnB,KAAKU,kBAAkB,KAAK,oBAAoB,YAAY,GAAG,EAAE,CAAC;EAElE,IAAI,KAAKV,YAAY,SAAS,GAC5B,KAAKW,kBAAkB;CAE3B;CAEA,oBAAoB;EAClB,KAAKX,YAAY,MAAM;EAEvB,MAAM,WAAW,gBAAgB,KAAKM,KAAK,KAAKZ,IAAI,IAAI,EACtD,2BAA2B,KAC7B,CAAC;EACD,KAAK,IAAI,QAAQ,WAAW,KAAKA,IAAI,EAAE,GAAG;GACxC,IAAI,CAAC,KAAK,YACR,OAAO;IACL,GAAG;IACH,YAAY,CACV,GAAI,SAAS,IAAI,KAAK,IAAI,GAAG,UAAU,cAAc,CAAC,CACxD;GACF;GAEF,KAAKM,YAAY,IAAI,KAAK,MAAM,IAAI;EACtC;CACF;CAEA,WAAW,MAAc;EACvB,OAAO,KAAK,KAAKA,YAAY,IAAI,IAAI,GAAG,iBAAiB,MAAM;CACjE;CAEA,QACE,EAAC,KAAK,WACN,EAAC,YACW;EACZ,MAAM,aACJ,SAAS,OAAO,SAAS,SACrB,SAAS,OAAO,UAChB,KAAKY,WAAW,cAAc,QAAQ,CAAC,EAAE;EAC/C,IAAI,CAAC,YAAY,QACf,MAAM,IAAI,MACR,2BAA2B,SAAS,KAAK,wCAC3C;EAIF,IAAI,YAAY,WAAW,QACzB,OAAO;EAET,MAAM,MAAqC,CAAC;EAC5C,KAAK,MAAM,OAAO,YAChB,IAAI,OAAO,IAAI;EAEjB,OAAO;CACT;CAEA,cAAc,QAAuB;EACnC,MAAM,QAAQ,cAAc,OAAO,QAAQ;EAC3C,MAAM,YAAY,KAAKA,WAAW,KAAK;EACvC,MAAM,SAAS,QAAQ,OAAO,KAAK,WAAW,KAAKH,WAAW;EAE9D,KAAKI,QAAQ,OAAO;GAClB,GAAG,OAAO;IACT,2BAA2B,KAAKL;EACnC,CAAC;EAED,IAAI,OAAO,SAAS,OAAO,QAAQ,WAAW,GAQ5C;EAEF,MAAM,MAAM,KAAKM,QAAQ,QAAQ,MAAM;EACvC,KAAKC,UAAU,OAAO,KAAK,qBAAqB,OAAO,KAAK,SAAS,CAAC;CACxE;CAEA,QAAQ,OAAe,KAAc;EACnC,MAAM,UAAU,OAAO,KAAK,GAAG,EAAE,KAAI,MAAK,GAAG,CAAC,CAAC;EAC/C,KAAKrB,IAAI,IACP;+BACyB,GAAG,KAAK,EAAE,IAAI,QAAQ,KAAK,GAAG,EAAE;kBAC7C,MAAM,KAAK,EAAC,QAAQ,QAAQ,OAAM,CAAC,EAAE,KAAK,GAAG,EAAE,KAAK,GAAG,EAAE;SAErE,OAAO,OAAO,GAAG,CACnB;CACF;CAcA,cAAc,QAAuB;EACnC,MAAM,QAAQ,cAAc,OAAO,QAAQ;EAC3C,MAAM,YAAY,KAAKkB,WAAW,KAAK;EACvC,MAAM,SAAS,QAAQ,OAAO,KAAK,WAAW,KAAKH,WAAW;EAC9D,MAAM,MAAM;GAAC,GAAG,OAAO;IAAM,2BAA2B,KAAKD;EAAQ;EAGrE,MAAM,SAAS,OAAO,MAClB,KAAKM,QACH,QAAQ,OAAO,KAAK,KAAKF,WAAW,KAAK,GAAG,KAAKH,WAAW,GAC5D,MACF,IACA;EACJ,MAAM,SAAS,KAAKK,QAAQ,QAAQ,MAAM;EAE1C,IAAI,QACF,KAAKE,aAAa,OAAO,QAAQ,UAAU,WAAW;EAExD,KAAKD,UAAU,OAAO,QAAQ,qBAAqB,OAAO,KAAK,SAAS,CAAC;EAEzE,MAAM,UAAU,UAAU;EAC1B,MAAM,QAAQ,OAAO,KAAK,OAAO,EAAE,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,GAAG;EAC5D,MAAM,WAAW,OAAO,KAAK,GAAG,EAAE,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,GAAG;EAE3D,IAAI;GACF,MAAM,EAAC,YAAW,KAAKrB,IAAI,IACzB;iBACS,GAAG,KAAK,EAAE;gBACX,SAAS,KAAK,GAAG,EAAE;kBACjB,MAAM,KAAK,OAAO,EAAE;WAE9B,CAAC,GAAG,OAAO,OAAO,GAAG,GAAG,GAAG,OAAO,OAAO,OAAO,CAAC,CACnD;GAIA,IAAI,YAAY,GACd,KAAKmB,QAAQ,OAAO,GAAG;EAE3B,SAAS,GAAG;GACV,IACE,EAAE,aAAa,eACf,EAAE,YAAY,0BAEd,MAAM;GAGR,MAAM,UAAU,CAAC,GAAG,OAAO,QAAQ,GAAG,GAAG,GAAG,OAAO,QAAQ,OAAO,CAAC,EAChE,KACE,CAAC,QAAQ,WACR,CAAC,QAAQ,0BAA0B,KAAK,CAAC,CAC7C,EACC,MAAM,GAAG,iBAAiB,WAAW;GACxC,MAAM,cACJ,iBAAiB,OAAO,YACxB,OAAO,OAAO,SAAS,gBAAgB,WACnC,gBAAgB,OAAO,SAAS,gBAChC;GACN,MAAM,QAAQ,IAAI,MAChB,uCAAuC,KAAKL,WAAW,YAAY,SAAS,OAAO,SAAS,OAAO,GAAG,OAAO,SAAS,KAAK,UAAU,UAAU,MAAM,YAAY,UAAU,KAAK,IAAI,QAAQ,OAAO,MACnM,EAAC,OAAO,EAAC,CACX;GACA,MAAM,OAAO;GACb,MAAM;EACR;CACF;CAEA,cAAc,KAAoB;EAChC,MAAM,QAAQ,cAAc,IAAI,QAAQ;EACxC,MAAM,YAAY,KAAKI,WAAW,KAAK;EACvC,MAAM,SAAS,KAAKE,QAClB,QAAQ,IAAI,KAAK,WAAW,KAAKL,WAAW,GAC5C,GACF;EAEA,KAAKQ,QAAQ,OAAO,MAAM;EAC1B,KAAKD,aAAa,OAAO,QAAQ,UAAU,WAAW;CACxD;CAEA,QAAQ,OAAe,QAAoB;EACzC,MAAM,QAAQ,OAAO,KAAK,MAAM,EAAE,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,GAAG;EAC3D,KAAKtB,IAAI,IACP,eAAe,GAAG,KAAK,EAAE,SAAS,MAAM,KAAK,OAAO,KACpD,OAAO,OAAO,MAAM,CACtB;CACF;CAEA,gBAAgB,UAA2B;EACzC,KAAK,MAAM,YAAY,SAAS,WAAW;GACzC,MAAM,QAAQ,cAAc,QAAQ;GAEpC,KAAKA,IAAI,IAAI,eAAe,GAAG,KAAK,GAAG;GAGvC,KAAKwB,eAAe,KAAK;EAC3B;CACF;CAEA,mBAAmB,QAAqB;EACtC,IAAI,OAAO,UACT,KAAKtB,eAAe,oBAAoB,OAAO,MAAM,OAAO,QAAQ;EAEtE,KAAKC,aAAa,MAAM,MAAM;EAC9B,MAAM,QAAQ,kBAAkB,OAAO,IAAI;EAC3C,KAAKH,IAAI,GAAG,KAAK,yBAAyB,KAAK,CAAC;EAGhD,KAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,OAAO,KAAK,OAAO,GACjE,KAAKgB,gBAAgB,OACnB,MAAM,MACN,SACA,SACA,OAAO,WAAW,QACpB;EAGF,KAAKS,YAAY,MAAM,IAAI;EAC3B,KAAKb,IAAI,OAAO,OAAO,KAAK,MAAM,IAAI;CACxC;CAEA,qBAAqB,KAA0B;EAC7C,KAAKV,eAAe,oBAAoB,IAAI,OAAO,IAAI,GAAG;EAO1D,KAAKC,aAAa,MAAM,GAAG;CAC7B;CAEA,mBAAmB,QAAqB;EACtC,KAAKD,eAAe,OAAO,OAAO,KAAK,OAAO,GAAG;EACjD,KAAKC,aAAa,MAAM,MAAM;EAE9B,MAAM,UAAU,cAAc,OAAO,GAAG;EACxC,MAAM,UAAU,cAAc,OAAO,GAAG;EACxC,KAAKH,IAAI,GAAG,KAAK,eAAe,GAAG,OAAO,EAAE,aAAa,GAAG,OAAO,GAAG;EAGtE,KAAKgB,gBAAgB,YAAY,SAAS,OAAO;EAEjD,KAAKU,cAAc,OAAO,GAAG;EAC7B,KAAKD,YAAY,OAAO;EACxB,KAAKb,IAAI,OAAO,OAAO,KAAK,SAAS,OAAO;CAC9C;CAEA,iBAAiB,KAAgB;EAC/B,IAAI,IAAI,eACN,KAAKV,eAAe,oBAAoB,IAAI,OAAO,IAAI,aAAa;EAEtE,KAAKC,aAAa,MAAM,GAAG;EAC3B,MAAM,QAAQ,cAAc,IAAI,KAAK;EACrC,MAAM,EAAC,SAAQ,IAAI;EACnB,MAAM,OAAO,wBAAwB,OAAO,IAAI,MAAM;EACtD,KAAKH,IAAI,GAAG,KACV,eAAe,GAAG,KAAK,EAAE,OAAO,GAAG,IAAI,EAAE,GAAG,cAAc,IAAI,GAChE;EAGA,KAAKgB,gBAAgB,OAAO,OAAO,MAAM,IAAI,OAAO,MAAM,IAAI,QAAQ;EAEtE,IAAI,IAAI,UACN,KAAKC,kBAAkB;OAIvB,KAAKS,cAAc,IAAI,KAAK;EAE9B,KAAKd,IAAI,OAAO,IAAI,KAAK,OAAO,IAAI,MAAM;CAC5C;CAEA,oBAAoB,KAAmB;EAGrC,KAAKT,aAAa,MAAM,GAAG;EAE3B,MAAM,QAAQ,cAAc,IAAI,KAAK;EACrC,IAAI,UAAU,IAAI,IAAI;EACtB,MAAM,UAAU,IAAI,IAAI;EAExB,MAAM,qBAAqB,sBACzB,IAAI,IAAI,MACR,IAAI,IAAI,IACV;EAMA,IAAI,YAAY,WAAW,CAAC,oBAAoB;GAC9C,KAAKa,gBAAgB,OACnB,OACA,IAAI,IAAI,MACR,IAAI,IAAI,MACR,IAAI,IAAI,IACV;GACA,IAAI,QAAQ,IAAI,IAAI,KAAK,OAAO,MAAM,QAAQ,IAAI,IAAI,KAAK,OAAO,GAChE,KAAKU,cAAc,IAAI,KAAK;GAE9B,KAAKd,IAAI,OAAO,IAAI,KAAK,yBAAyB,IAAI,KAAK,IAAI,GAAG;GAClE;EACF;EAGA,IAAI,oBAAoB;GAWtB,MAAM,cAAc,wBAClB,OACA,IAAI,KACJ,gBACF;GACA,IAAI,YAAY,SAAS;IAIvB,KAAKZ,IAAI,GAAG,KACV,eAAe,GAAG,KAAK,EAAE,UAAU,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,GACjE;IACA,UAAU;GACZ;GACA,MAAM,YAAY,KAChB,WAAW,KAAKA,IAAI,IAAI,OAAO,KAAK,EAAE,MACpC,cAAa,UAAU,SAAS,KAClC,CACF;GAIA,MAAM,WAAW,KAAKA,IAAI,GACvB,QACC;;4BAGF,EACC,IAAmB,KAAK,EACxB,KAAK,EAAC,UAAS,GAAG;GACrB,MAAM,WAAW,OAAO;GACxB,MAAM,UAAU,OAAO,KAAK,UAAU,OAAO;GAC7C,MAAM,QAAQ;IACZ,yBAAyB;KACvB,GAAG;KACH,MAAM;KACN,SAAS,WAAW,UAAU,UAAU,QAAQ,SAAS,CACvD,QACA,WAAW,UAAU;MAAC,GAAG;MAAa,KAAK,KAAK;KAAG,IAAI,IACzD,CAAC;IACH,CAAC;IACD,eAAe,GAAG,QAAQ,EAAE,IAAI,QAAQ,IAAI,EAAE,EAAE,KAAK,GAAG,EAAE;kBAChD,QAAQ,IAAI,EAAE,EAAE,KAAK,GAAG,EAAE,QAAQ,GAAG,KAAK,EAAE;IACtD,cAAc,GAAG,KAAK,EAAE;IACxB,eAAe,GAAG,QAAQ,EAAE,aAAa,GAAG,KAAK,EAAE;IACnD,GAAG,SAAS,KAAI,QAAO,GAAG,IAAI,EAAE;GAClC;GACA,KAAKA,IAAI,GAAG,KAAK,MAAM,KAAK,EAAE,CAAC;EACjC;EACA,IAAI,YAAY,SACd,KAAKA,IAAI,GAAG,KACV,eAAe,GAAG,KAAK,EAAE,UAAU,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,GACjE;EAIF,KAAKgB,gBAAgB,OACnB,OACA,IAAI,IAAI,MACR,IAAI,IAAI,MACR,IAAI,IAAI,IACV;EAEA,KAAKU,cAAc,IAAI,KAAK;EAC5B,KAAKd,IAAI,OAAO,IAAI,KAAK,OAAO,IAAI,GAAG;CACzC;CAEA,kBAAkB,KAAiB;EACjC,MAAM,QAAQ,cAAc,IAAI,KAAK;EACrC,MAAM,EAAC,WAAU;EACjB,KAAKZ,IAAI,GAAG,KAAK,eAAe,GAAG,KAAK,EAAE,QAAQ,GAAG,MAAM,GAAG;EAG9D,KAAKgB,gBAAgB,aAAa,OAAO,MAAM;EAC/C,KAAKb,aAAa,MAAM,GAAG;EAE3B,KAAKuB,cAAc,IAAI,KAAK;EAC5B,KAAKd,IAAI,OAAO,IAAI,KAAK,OAAO,MAAM;CACxC;CAEA,iBAAiB,MAAiB;EAChC,KAAKV,eAAe,KAAK,KAAK,EAAE;EAChC,KAAKC,aAAa,MAAM,IAAI;EAE5B,MAAM,OAAO,cAAc,KAAK,EAAE;EAClC,KAAKH,IAAI,GAAG,KAAK,wBAAwB,GAAG,IAAI,GAAG;EAGnD,KAAKgB,gBAAgB,YAAY,IAAI;EAErC,KAAKS,YAAY,IAAI;EACrB,KAAKb,IAAI,OAAO,KAAK,KAAK,IAAI;CAChC;CAEA,mBAAmB,QAAqB;EACtC,MAAM,QAAQ,uBAAuB,OAAO,IAAI;EAChD,KAAKZ,IAAI,GAAG,KAAK,yBAAyB,KAAK,CAAC;EAIhD,KAAKyB,YAAY,MAAM,SAAS;EAChC,KAAKb,IAAI,OAAO,OAAO,KAAK,MAAM,IAAI;CACxC;CAEA,iBAAiB,MAAiB;EAChC,MAAM,OAAO,cAAc,KAAK,EAAE;EAClC,KAAKZ,IAAI,GAAG,KAAK,wBAAwB,GAAG,IAAI,GAAG;EACnD,KAAKY,IAAI,OAAO,KAAK,KAAK,IAAI;CAChC;CAEA,cAAc,OAAmB;EAC/B,KAAKV,eAAe,iBAAiB,OAAO,KAAKY,QAAQ;EACzD,KAAKW,YAAY,cAAc,KAAK,CAAC;CACvC;;;;CAKA,UACE,OACA,KACA,mBACA;EAIA,IAAI,KAAKrB,UAAU,aAAa,sBAAsB,KAAA,GAAW;GAC/D,KAAKH,WAAW,SACd,KAAKa,UACL,KAAKa,QACL,OACA,KACA,iBACF;GACA,KAAKC;EACP;CACF;CAEA,aAAa,OAAe,KAAiB,aAAwB;EAInE,IAAI,KAAKxB,UAAU,aAAa,aAAa,QAAQ;GACnD,KAAKH,WAAW,YAAY,KAAKa,UAAU,KAAKa,QAAQ,OAAO,GAAG;GAClE,KAAKC;EACP;CACF;CAEA,eAAe,OAAe;EAC5B,IAAI,KAAKxB,UAAU,WAAW;GAC5B,KAAKH,WAAW,cAAc,KAAKa,UAAU,KAAK;GAClD,KAAKc;EACP;CACF;CAEA,YAAY,OAAe;EACzB,KAAKC,iBAAiB;EACtB,IAAI,KAAKzB,UAAU,WAAW;GAC5B,KAAKH,WAAW,WAAW,KAAKa,UAAU,KAAK;GAC/C,KAAKc;EACP;EACA,KAAKX,kBAAkB;CACzB;CAEA,gBAAgB,EACd,UACA,WACA,SACA,WACA,iBACkB;EAClB,MAAM,YAAY,cAAc,QAAQ;EACxC,MAAM,YAAY,KAAK,KAAKX,YAAY,IAAI,SAAS,CAAC;EACtD,MAAM,aAAa,SAAS,OAAO;EACnC,MAAM,OAAO,CAAC,GAAG,YAAY,GAAG,OAAO;EAcvC,MAAM,EAAC,UAAU,YAAW,KAAKwB,uBAC/B,UACA,OACA,QAAO,eAAe,KAAK,aAAa,CAC1C;EACA,IAAI,SAAS,WAAW,GAAG;GACzB,KAAKlB,IAAI,QACP,QAAQ,SACJ,wBAAwB,UAAU,OAAO,aACpC,UAAU,iDACT,QAAQ,KAAK,GAAG,EAAE,KACxB,kCAAkC,UAAU,OAAO,aAC9C,UAAU,mCACnB,aACF;GACA;EACF;EACA,IAAI,QAAQ,QACV,KAAKA,IAAI,QACP,eAAe,UAAU,mDACR,QAAQ,KAAK,GAAG,EAAE,IACnC,aACF;EAEF,MAAM,cAAc,IAAI,IAAI,QAAQ;EAGpC,MAAM,gBAAgB,CAAC,GAAG,MAAM,wBAAwB,EAAE,IAAI,EAAE,EAAE,KAAK,GAAG;EAC1E,MAAM,SAAS,MAAM,KAAK,EAAC,QAAQ,KAAK,SAAS,EAAC,CAAC,EAChD,KAAK,GAAG,EACR,KAAK,GAAG;EACX,MAAM,gBAAgB,WAAW,IAAI,EAAE,EAAE,KAAK,GAAG;EAEjD,IAAI,aAAa;EACjB,IAAI,UAAU;EACd,KAAK,MAAM,KAAK,WAAW;GACzB,MAAM,MAAM,QACV,OAAO,YAAY,KAAK,KAAK,GAAG,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAChD,WACA,KAAKG,WACP;GACA,MAAM,SAAS,KAAKK,QAAQ,KAAK,EAAC,SAAQ,CAAC;GAC3C,MAAM,QAAQ,KAAKnB,WAAW,eAAe,WAAW,MAAM;GAC9D,IAAI,OAAO,OAAA,OAAiB,MAAM,eAAe,WAAW;IAC1D;IACA;GACF;GAGA,MAAM,WACJ,OAAO,OAAA,MACH,KAAK,QACH,OAAM,MAAM,0BAA0B,MAAM,OAAO,SACrD,IACA,MACJ,QAAO,MAAK,YAAY,IAAI,CAAC,CAAC;GAChC,IAAI,QAAQ,WAAW,GAAG;IAExB;IACA;GACF;GACA,MAAM,cAAc,QAAQ,KAAI,QAAO,GAAG,GAAG,GAAG,EAAE,YAAY,GAAG,GAAG,GAAG;GACvE,KAAKD,IAAI,IACC;sBACM,GAAG,SAAS,EAAE,IAAI,cAAc,YAAY,OAAO;yBAChD,cAAc;0BACb,YAAY,KAAK,GAAG,EAAE;SAExC,GAAG,OAAO,OAAO,IAAI,GAAG,GACxB,SACF;GACA;EACF;EAEA,KAAKY,IAAI,QACP,cAAc,WAAW,iBAAiB,QAAQ,SAAS,WAC7D;CACF;CAEA;;;;;;;CAQA,uBACE,UACA,MACA,QACyC;EACzC,MAAM,UAAU,KAAKT,aAAa,YAAY,QAAQ;EACtD,MAAM,WAAqB,CAAC;EAC5B,MAAM,UAAoB,CAAC;EAC3B,KAAK,MAAM,OAAO,MAAM;GACtB,IAAI,CAAC,QAAQ,IAAI,GAAG,GAClB;GAEF,MAAM,WAAW,QAAQ,IAAI,GAAG;GAChC,MAAM,SAAS,OAAO,QAAQ;GAC9B,IAAI,CAAC,OAAO,QAAQ;IAClB,QAAQ,KAAK,GAAG;IAChB;GACF;GACA,SAAS,KAAK,GAAG;GACjB,IAAI,CAAC,UAAU,OAAO,UAAU,QAAQ,GACtC,KAAKA,aAAa,YAAY,UAAU,KAAK,OAAO,QAAQ;EAEhE;EACA,OAAO;GAAC;GAAU;EAAO;CAC3B;CAEA,yBAAyB,KAAwB;EAC/C,MAAM,EAAC,UAAU,SAAS,QAAQ,kBAAiB;EACnD,MAAM,YAAY,cAAc,QAAQ;EACxC,MAAM,aAAa,SAAS,OAAO;EAKnC,MAAM,EAAC,UAAU,YAAW,KAAK2B,uBAC/B,UACA,CAAC,GAAG,YAAY,GAAG,OAAO,IAC1B,QACE,eACE,KACA,gBAAgB,EAAC,UAAU,cAAc,SAAQ,IAAI,KAAA,CACvD,CACJ;EACA,IAAI,QAAQ,QACV,KAAKlB,IAAI,OACP,kCAAkC,UAAU,gBACtC,QAAQ,KAAK,GAAG,EAAE,6CACxB,aACF;EAQF,IAAI,SAAS,WAAW,GAAG;GACzB,IAAI,QAAQ,WAAW,GACrB,KAAKA,IAAI,QACP,6CAA6C,UAAU,iCAEzD;GAEF;EACF;EACA,MAAM,aAAa,IAAI,IAAI,OAAO;EAClC,MAAM,OAAO,CACX,GAAG,WAAW,QAAO,MAAK,CAAC,WAAW,IAAI,CAAC,CAAC,GAC5C,GAAG,QAAQ,QAAO,MAAK,CAAC,WAAW,IAAI,CAAC,CAAC,CAC3C;EAEA,MAAM,iBAAiB,KAAK,oBAAoB,YAAY,KAAKZ,IAAI,EAAE,CAAC;EACxE,KAAK,MAAM,OAAO,MAChB,eAAe,iBAAiB,WAAW,GAAG;EAEhD,KAAKG,aAAa,SAAS,UAAU,IAAI;EAGzC,KAAKuB,cAAc,QAAQ;EAC3B,IAAI,QACF,KAAKK,qBAAqB;GAAC,OAAO;GAAW,SAAS;GAAM,GAAG;EAAM;EAEvE,KAAKnB,IAAI,OAAO,wBAAwB,WAAW;CAWrD;CAEA,cAAc,QAAuB,WAAiC;EACpE,IAAI,cAAc,KAAKE,UACrB,MAAM,IAAI,MACR,oBAAoB,UAAU,kCAC5B,KAAKA,SACN,IAAI,UAAU,MAAM,GACvB;EAEF,2BAA2B,KAAKd,KAAK,SAAS;EAE9C,IAAI,KAAK6B,gBAAgB;GACvB,MAAM,QAAQ,KAAK,IAAI;GACvB,KAAK7B,IAAI,GAAG,OAAO,UAAU;GAC7B,KAAKY,IAAI,OACP,yCAAyC,KAAK,IAAI,IAAI,MAAM,KAC9D;EACF;EAEA,IAAI,KAAKR,UAAU,gBACjB,KAAKJ,IAAI,OAAO;EAGlB,MAAM,YAAY,KAAK,IAAI,IAAI,KAAKa;EACpC,KAAKD,IAAI,QAAQ,gBAAgB,KAAKE,SAAS,IAAI,UAAU,KAAK;EAElE,OAAO;GACL;GACA,mBAAmB,KAAKiB;GACxB,eAAe,KAAKF;GACpB,kBAAkB,KAAKD,uBAAuB;GAC9C,sBAAsB,OAAO;EAC/B;CACF;CAEA,MAAM,IAAgB;EACpB,GAAG,OAAO,wBAAwB,KAAKd,UAAU;EACjD,KAAKd,IAAI,SAAS;EAIlB,KAAKM,YAAY,MAAM;CACzB;AACF;AAEA,SAAS,sBACP,SACA,SACS;CACT,OACE,QAAQ,aAAa,QAAQ,YAC7B,aAAa,OAAO,MAAM,aAAa,OAAO,KAC9C,cAAc,OAAO,MAAM,cAAc,OAAO;AAEpD;AAEA,SAAS,qBACP,KACA,EAAC,eACqB;CACtB,IAAI,CAAC,aAAa,QAChB;CAEF,OAAO,YAAY,QAAO,QAAO,OAAO,GAAG;AAC7C;AAEA,SAAS,0BAA0B,OAA0C;CAC3E,IAAI,OAAO,UAAU,UACnB,OAAO,OAAO,OAAO,IAAI,KAAK,MAAM,QAChC,qCACA,KAAA;CAGN,MAAM,YACJ,OAAO,UAAU,WACb,OAAO,WAAW,KAAK,IACvB,iBAAiB,aACf,MAAM,aACN,KAAA;CACR,IAAI,cAAc,KAAA,KAAa,YAAY,uBAEzC,OAAO,aADW,OAAO,UAAU,WAAW,WAAW,SAC3B,aAAa,UAAU,cAAc;AAIvE;;AAGA,SAAS,QAAQ,IAAyB;CACxC,IAAI;EACF,GAAG;EACH;CACF,SAAS,GAAG;EACV,OAAO,qBAAK,IAAI,MAAM,iBAAiB;CACzC;AACF;AAEA,SAAS,YAAY,KAAqB;CACxC,IAAI,eAAe,OACjB,OAAO;CAET,MAAM,wBAAQ,IAAI,MAAM;CACxB,MAAM,QAAQ;CACd,OAAO;AACT"}