@rocicorp/zero 1.11.0-canary.3 → 1.11.0-canary.30

package/out/zero-cache/src/services/replicator/change-log-cookies.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-cookies.js","names":["#upsertMetadata","#upsertBackfill","#renameMetadataTable","#renameBackfillTable","#deleteMetadataTable","#deleteBackfillTable","#renameBackfillColumn","#deleteBackfillColumn","#run"],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-cookies.ts"],"sourcesContent":["/**\n * The change log's *second* job.\n *\n * A change log is a buffer **and** a cookie jar. The buffer catches a\n * reconnecting subscriber up to the head; the cookie jar holds the backfill\n * progress state that the change-streamer hands the change-source on every\n * `startStream`, so that the change source never has to persist backfill state\n * upstream. See `change-log-db.ts` for the buffer.\n *\n * There are two cookies, mirroring the two `cdc` tables the Postgres change log\n * has always carried (`change-streamer/schema/tables.ts`):\n *\n * - **table metadata**, the most current {@link TableMetadata} of a table, which\n *   a {@link BackfillRequest} has to carry; and\n * - **backfilling columns**, the opaque {@link BackfillID} of every column whose\n *   backfill has not completed.\n *\n * Both are a *fold over the change stream*: the cookie set is only meaningful\n * paired with the watermark it was folded up to. {@link cookieOps} is that fold,\n * expressed transport-free so that the Postgres change log and the SQLite change\n * log cannot drift from each other — they must agree on all eight transitions\n * forever, and the failure mode if they disagree is a backfill that is silently\n * never re-requested. The two interpreters are {@link ChangeLogCookieWriter}\n * here and `Storer.#trackBackfillMetadata()` in\n * `change-streamer/storer.ts`.\n *\n * The tables live in the change-log database rather than in the replica: the\n * replicator is a subscriber *downstream* of the change-streamer's write loop,\n * so its state version trails the log's head, and pairing its cookies with the\n * log's resume watermark would lose every backfill that completed in between.\n * The replica's own copies (`_zero.tableMetadata`, `_zero.column_metadata`) are\n * the *seed* source; these are the *working* set.\n */\n\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Database, Statement} from '../../../../zqlite/src/db.ts';\nimport type {\n  BackfillID,\n  Identifier,\n  SchemaChange,\n  TableMetadata,\n} from '../change-source/protocol/current/data.ts';\nimport {\n  backfillRequestSchema,\n  type BackfillRequest,\n} from '../change-source/protocol/current/upstream.ts';\n\nexport const CHANGE_LOG_TABLE_METADATA_TABLE = '_zero.changeLogTableMetadata';\nexport const CHANGE_LOG_BACKFILLING_TABLE = '_zero.changeLogBackfilling';\n\n// The names follow the file's existing convention, beside\n// \"_zero.changeLogStream\" and \"_zero.changeLogMeta\".\n//\n// `metadata` and `backfill` hold the JSON that the `cdc` tables hold as JSONB.\n// SQLite has no JSONB, and nothing here queries into the documents: they are\n// stored to be handed back to the change source verbatim.\nexport const CREATE_CHANGE_LOG_COOKIE_SCHEMA = /*sql*/ `\n  CREATE TABLE \"${CHANGE_LOG_TABLE_METADATA_TABLE}\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"metadata\" TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n\n  CREATE TABLE \"${CHANGE_LOG_BACKFILLING_TABLE}\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n`;\n\nexport const DROP_CHANGE_LOG_COOKIE_TABLES = /*sql*/ `\n  DROP TABLE IF EXISTS \"${CHANGE_LOG_TABLE_METADATA_TABLE}\";\n  DROP TABLE IF EXISTS \"${CHANGE_LOG_BACKFILLING_TABLE}\";\n`;\n\nexport type TableMetadataCookie = {\n  readonly schema: string;\n  readonly table: string;\n  readonly metadata: TableMetadata;\n};\n\nexport type BackfillCookie = {\n  readonly schema: string;\n  readonly table: string;\n  readonly column: string;\n  readonly backfill: BackfillID;\n};\n\n/**\n * The whole cookie set, as of the log's head. Both lists are ordered by their\n * primary key so that two stores' sets can be compared directly.\n */\nexport type CookieSet = {\n  readonly tableMetadata: readonly TableMetadataCookie[];\n  readonly backfilling: readonly BackfillCookie[];\n};\n\nexport const EMPTY_COOKIE_SET: CookieSet = {\n  tableMetadata: [],\n  backfilling: [],\n};\n\n/**\n * A transport-free description of a cookie mutation. This is the whole of what\n * a {@link SchemaChange} means to the cookie jar; everything else about the\n * change belongs to the buffer.\n */\nexport type CookieOp =\n  | {op: 'upsert-metadata'; table: Identifier; metadata: TableMetadata}\n  | {\n      op: 'upsert-backfill';\n      table: Identifier;\n      column: string;\n      backfill: BackfillID;\n    }\n  | {op: 'rename-table'; old: Identifier; new: Identifier}\n  | {op: 'drop-table'; table: Identifier}\n  | {op: 'rename-column'; table: Identifier; old: string; new: string}\n  | {op: 'drop-column'; table: Identifier; column: string}\n  | {op: 'complete-backfill'; table: Identifier; columns: readonly string[]};\n\nexport type CookieOpTag = CookieOp['op'];\n\n/**\n * The fold, and the only place either store decides what a schema change means\n * for the cookie set. Both interpreters are mechanical translations of what this\n * returns, in the order it returns them.\n *\n * Changes that carry no cookie state — index changes, and the `create-table` /\n * `add-column` variants whose optional `metadata` and `backfill` fields are\n * absent — produce no ops.\n */\nexport function cookieOps(change: SchemaChange): CookieOp[] {\n  switch (change.tag) {\n    case 'update-table-metadata':\n      return [\n        {op: 'upsert-metadata', table: change.table, metadata: change.new},\n      ];\n\n    case 'create-table': {\n      const {spec, metadata, backfill} = change;\n      const table = {schema: spec.schema, name: spec.name};\n      const ops: CookieOp[] = [];\n      if (metadata) {\n        ops.push({op: 'upsert-metadata', table, metadata});\n      }\n      if (backfill) {\n        for (const [column, id] of Object.entries(backfill)) {\n          ops.push({op: 'upsert-backfill', table, column, backfill: id});\n        }\n      }\n      return ops;\n    }\n\n    case 'rename-table':\n      return [{op: 'rename-table', old: change.old, new: change.new}];\n\n    case 'drop-table':\n      return [{op: 'drop-table', table: change.id}];\n\n    case 'add-column': {\n      const {table, tableMetadata, column, backfill} = change;\n      const ops: CookieOp[] = [];\n      if (tableMetadata) {\n        ops.push({op: 'upsert-metadata', table, metadata: tableMetadata});\n      }\n      if (backfill) {\n        ops.push({\n          op: 'upsert-backfill',\n          table,\n          column: column.name,\n          backfill,\n        });\n      }\n      return ops;\n    }\n\n    case 'update-column': {\n      // Only a rename moves the cookie; the rest of a column update does not\n      // affect whether or how it is being backfilled.\n      const {table, old, new: updated} = change;\n      return old.name === updated.name\n        ? []\n        : [{op: 'rename-column', table, old: old.name, new: updated.name}];\n    }\n\n    case 'drop-column':\n      return [{op: 'drop-column', table: change.table, column: change.column}];\n\n    case 'backfill-completed': {\n      const {relation, columns} = change;\n      // The rowKey columns are excluded from `columns` but are backfilled with\n      // them, so both are cleared.\n      const cleared = [...relation.rowKey.columns, ...columns];\n      return cleared.length === 0\n        ? []\n        : [\n            {\n              op: 'complete-backfill',\n              table: {schema: relation.schema, name: relation.name},\n              columns: cleared,\n            },\n          ];\n    }\n\n    case 'create-index':\n    case 'drop-index':\n      return [];\n\n    default:\n      // Unreachable: `isSchemaChange()` gates on the same tag set this switch\n      // covers. It is exhaustive so that a new schema-change tag is a compile\n      // error here rather than a silently dropped backfill.\n      unreachable(change);\n  }\n}\n\n/**\n * The SQLite interpreter of {@link cookieOps}, over prepared statements on the\n * change-log database.\n *\n * The statements run inside the log's transaction, in stream order, at the point\n * the schema change is appended (see `ChangeLogStreamWriter`). That is what\n * makes the cookie set atomic with the head — a crash cannot leave cookies ahead\n * of or behind the rows they were folded from — and what makes an aborted\n * transaction discard its cookies with its rows, for free.\n *\n * Statements are prepared against the tables as they exist now, so an instance\n * does not survive a reconcile that drops and recreates them.\n */\nexport class ChangeLogCookieWriter {\n  readonly #upsertMetadata: Statement;\n  readonly #upsertBackfill: Statement;\n  readonly #renameMetadataTable: Statement;\n  readonly #renameBackfillTable: Statement;\n  readonly #deleteMetadataTable: Statement;\n  readonly #deleteBackfillTable: Statement;\n  readonly #renameBackfillColumn: Statement;\n  readonly #deleteBackfillColumn: Statement;\n\n  constructor(db: Database) {\n    this.#upsertMetadata = db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n        (\"schema\", \"table\", \"metadata\") VALUES (?, ?, ?)\n        ON CONFLICT (\"schema\", \"table\")\n        DO UPDATE SET \"metadata\" = excluded.\"metadata\"\n    `);\n    this.#upsertBackfill = db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n        (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n        ON CONFLICT (\"schema\", \"table\", \"column\")\n        DO UPDATE SET \"backfill\" = excluded.\"backfill\"\n    `);\n    this.#renameMetadataTable = db.prepare(/*sql*/ `\n      UPDATE \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n        SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n    `);\n    this.#renameBackfillTable = db.prepare(/*sql*/ `\n      UPDATE \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n        SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n    `);\n    this.#deleteMetadataTable = db.prepare(/*sql*/ `\n      DELETE FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ?\n    `);\n    this.#deleteBackfillTable = db.prepare(/*sql*/ `\n      DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ?\n    `);\n    this.#renameBackfillColumn = db.prepare(/*sql*/ `\n      UPDATE \"${CHANGE_LOG_BACKFILLING_TABLE}\" SET \"column\" = ?\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `);\n    this.#deleteBackfillColumn = db.prepare(/*sql*/ `\n      DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `);\n  }\n\n  /** Applies the change's cookie ops, returning the ops that were applied. */\n  apply(change: SchemaChange): CookieOp[] {\n    const ops = cookieOps(change);\n    for (const op of ops) {\n      this.#run(op);\n    }\n    return ops;\n  }\n\n  #run(op: CookieOp): void {\n    switch (op.op) {\n      case 'upsert-metadata':\n        this.#upsertMetadata.run(\n          op.table.schema,\n          op.table.name,\n          BigIntJSON.stringify(op.metadata),\n        );\n        break;\n\n      case 'upsert-backfill':\n        this.#upsertBackfill.run(\n          op.table.schema,\n          op.table.name,\n          op.column,\n          BigIntJSON.stringify(op.backfill),\n        );\n        break;\n\n      case 'rename-table':\n        this.#renameMetadataTable.run(\n          op.new.schema,\n          op.new.name,\n          op.old.schema,\n          op.old.name,\n        );\n        this.#renameBackfillTable.run(\n          op.new.schema,\n          op.new.name,\n          op.old.schema,\n          op.old.name,\n        );\n        break;\n\n      case 'drop-table':\n        this.#deleteMetadataTable.run(op.table.schema, op.table.name);\n        this.#deleteBackfillTable.run(op.table.schema, op.table.name);\n        break;\n\n      case 'rename-column':\n        this.#renameBackfillColumn.run(\n          op.new,\n          op.table.schema,\n          op.table.name,\n          op.old,\n        );\n        break;\n\n      case 'drop-column':\n        this.#deleteBackfillColumn.run(\n          op.table.schema,\n          op.table.name,\n          op.column,\n        );\n        break;\n\n      case 'complete-backfill':\n        // A per-column delete rather than an `IN` list, which cannot be a\n        // single prepared statement across arities. The columns of one\n        // completed backfill are few, and this runs only on schema changes.\n        for (const column of op.columns) {\n          this.#deleteBackfillColumn.run(\n            op.table.schema,\n            op.table.name,\n            column,\n          );\n        }\n        break;\n\n      default:\n        unreachable(op);\n    }\n  }\n}\n\nconst tableKey = (schema: string, table: string) =>\n  JSON.stringify([schema, table]);\n\nconst columnKey = (schema: string, table: string, column: string) =>\n  JSON.stringify([schema, table, column]);\n\nconst cmp = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0);\n\n/**\n * The in-memory interpreter of {@link cookieOps}: the third one, and the only\n * one that is not a store.\n *\n * A replica-derived cookie set is a snapshot at the replica's state version,\n * which trails the change log's head, so it cannot be compared with a\n * Postgres-derived set directly. This carries it forward over the schema changes\n * the log holds in between, which turns \"the two stores disagree, but they are\n * also at different positions\" into an exact equality — and makes a\n * disagreement between any two of the three interpreters visible as an\n * inequality that neither store would have noticed on its own.\n *\n * `changes` must be the log's schema changes over `(from, to]` in stream order.\n * Anything else produces a set that belongs to no watermark.\n */\nexport function foldCookies(\n  cookies: CookieSet,\n  changes: Iterable<SchemaChange>,\n): CookieSet {\n  const metadata = new Map<string, TableMetadataCookie>(\n    cookies.tableMetadata.map(c => [tableKey(c.schema, c.table), c]),\n  );\n  const backfilling = new Map<string, BackfillCookie>(\n    cookies.backfilling.map(c => [columnKey(c.schema, c.table, c.column), c]),\n  );\n  const dropColumn = ({schema, name}: Identifier, column: string) =>\n    backfilling.delete(columnKey(schema, name, column));\n\n  for (const change of changes) {\n    for (const op of cookieOps(change)) {\n      switch (op.op) {\n        case 'upsert-metadata':\n          metadata.set(tableKey(op.table.schema, op.table.name), {\n            schema: op.table.schema,\n            table: op.table.name,\n            metadata: op.metadata,\n          });\n          break;\n\n        case 'upsert-backfill':\n          backfilling.set(\n            columnKey(op.table.schema, op.table.name, op.column),\n            {\n              schema: op.table.schema,\n              table: op.table.name,\n              column: op.column,\n              backfill: op.backfill,\n            },\n          );\n          break;\n\n        case 'rename-table': {\n          const {old, new: renamed} = op;\n          const moved = metadata.get(tableKey(old.schema, old.name));\n          if (moved) {\n            metadata.delete(tableKey(old.schema, old.name));\n            metadata.set(tableKey(renamed.schema, renamed.name), {\n              ...moved,\n              schema: renamed.schema,\n              table: renamed.name,\n            });\n          }\n          // Snapshotted before mutating, since the rename both deletes and\n          // inserts into the map being scanned.\n          for (const [key, cookie] of [...backfilling]) {\n            if (cookie.schema === old.schema && cookie.table === old.name) {\n              backfilling.delete(key);\n              backfilling.set(\n                columnKey(renamed.schema, renamed.name, cookie.column),\n                {...cookie, schema: renamed.schema, table: renamed.name},\n              );\n            }\n          }\n          break;\n        }\n\n        case 'drop-table': {\n          const {schema, name} = op.table;\n          metadata.delete(tableKey(schema, name));\n          for (const [key, cookie] of [...backfilling]) {\n            if (cookie.schema === schema && cookie.table === name) {\n              backfilling.delete(key);\n            }\n          }\n          break;\n        }\n\n        case 'rename-column': {\n          const {schema, name} = op.table;\n          const moved = backfilling.get(columnKey(schema, name, op.old));\n          if (moved) {\n            backfilling.delete(columnKey(schema, name, op.old));\n            backfilling.set(columnKey(schema, name, op.new), {\n              ...moved,\n              column: op.new,\n            });\n          }\n          break;\n        }\n\n        case 'drop-column':\n          dropColumn(op.table, op.column);\n          break;\n\n        case 'complete-backfill':\n          for (const column of op.columns) {\n            dropColumn(op.table, column);\n          }\n          break;\n\n        default:\n          unreachable(op);\n      }\n    }\n  }\n\n  return {\n    tableMetadata: [...metadata.values()].toSorted(\n      (a, b) => cmp(a.schema, b.schema) || cmp(a.table, b.table),\n    ),\n    backfilling: [...backfilling.values()].toSorted(\n      (a, b) =>\n        cmp(a.schema, b.schema) ||\n        cmp(a.table, b.table) ||\n        cmp(a.column, b.column),\n    ),\n  };\n}\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\n/**\n * Returns one {@link BackfillRequest} for each table with an active backfill.\n * Each request contains the stored table metadata, if available.\n */\nexport function backfillRequestsFrom(cookies: CookieSet): BackfillRequest[] {\n  const metadata = new Map(\n    cookies.tableMetadata.map(c => [tableKey(c.schema, c.table), c.metadata]),\n  );\n  const requests: BackfillRequest[] = [];\n  let curr: BackfillRequest | undefined;\n  for (const {schema, table, column, backfill} of cookies.backfilling) {\n    if (curr?.table.schema !== schema || curr.table.name !== table) {\n      curr = {\n        table: {\n          schema,\n          name: table,\n          // Use `null` when a backfilling table has no metadata. This matches\n          // the result of the Postgres left join.\n          metadata: metadata.get(tableKey(schema, table)) ?? null,\n        },\n        columns: {},\n      };\n      requests.push(curr);\n    }\n    curr.columns[column] = backfill;\n  }\n  return v.parse(requests, backfillRequestsSchema);\n}\n\n/**\n * Reads the whole cookie set, ordered by primary key. This is what a\n * change-streamer would hand `startStream`, and what the Postgres-derived and\n * replica-derived sets are compared against.\n */\nexport function readCookies(db: Database): CookieSet {\n  const tableMetadata = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"metadata\"\n        FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n        ORDER BY \"schema\", \"table\"\n    `)\n    .all<{schema: string; table: string; metadata: string}>()\n    .map(({schema, table, metadata}) => ({\n      schema,\n      table,\n      metadata: BigIntJSON.parse(metadata) as TableMetadata,\n    }));\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"column\", \"backfill\"\n        FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n        ORDER BY \"schema\", \"table\", \"column\"\n    `)\n    .all<{schema: string; table: string; column: string; backfill: string}>()\n    .map(({schema, table, column, backfill}) => ({\n      schema,\n      table,\n      column,\n      backfill: BigIntJSON.parse(backfill) as BackfillID,\n    }));\n\n  return {tableMetadata, backfilling};\n}\n\n/**\n * Replaces the cookie set wholesale.\n *\n * The cookies are unversioned point-in-time state, so a reconciliation that\n * moves the head invalidates them: truncating the stream above `W` does not roll\n * the cookie set back to `W`, and the truncated transactions can contain\n * `create-table`, `add-column`, and `backfill-completed`. The caller must run\n * this in the same transaction as the reconciliation that invalidated them.\n */\nexport function replaceCookies(db: Database, cookies: CookieSet): void {\n  db.prepare(/*sql*/ `DELETE FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"`).run();\n  db.prepare(/*sql*/ `DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"`).run();\n\n  if (cookies.tableMetadata.length > 0) {\n    const insert = db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n        (\"schema\", \"table\", \"metadata\") VALUES (?, ?, ?)\n    `);\n    for (const {schema, table, metadata} of cookies.tableMetadata) {\n      insert.run(schema, table, BigIntJSON.stringify(metadata));\n    }\n  }\n  if (cookies.backfilling.length > 0) {\n    const insert = db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n        (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n    `);\n    for (const {schema, table, column, backfill} of cookies.backfilling) {\n      insert.run(schema, table, column, BigIntJSON.stringify(backfill));\n    }\n  }\n}\n\n/** Retained cookie rows, by table. Both are normally zero. */\nexport type CookieRowCounts = {\n  readonly tableMetadata: number;\n  readonly backfilling: number;\n};\n\nexport function readCookieRowCounts(db: Database): CookieRowCounts {\n  return db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT count(*) FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\")\n          AS \"tableMetadata\",\n        (SELECT count(*) FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\")\n          AS \"backfilling\"\n    `)\n    .get<CookieRowCounts>();\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,IAAa,kCAAkC;AAC/C,IAAa,+BAA+B;AAQ5C,IAAa,kCAA0C;kBACrC,gCAAgC;;;;;;;kBAOhC,6BAA6B;;;;;;;;AAS/C,IAAa,gCAAwC;0BAC3B,gCAAgC;0BAChC,6BAA6B;;;;;;;;;;;AA4DvD,SAAgB,UAAU,QAAkC;CAC1D,QAAQ,OAAO,KAAf;EACE,KAAK,yBACH,OAAO,CACL;GAAC,IAAI;GAAmB,OAAO,OAAO;GAAO,UAAU,OAAO;EAAG,CACnE;EAEF,KAAK,gBAAgB;GACnB,MAAM,EAAC,MAAM,UAAU,aAAY;GACnC,MAAM,QAAQ;IAAC,QAAQ,KAAK;IAAQ,MAAM,KAAK;GAAI;GACnD,MAAM,MAAkB,CAAC;GACzB,IAAI,UACF,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO;GAAQ,CAAC;GAEnD,IAAI,UACF,KAAK,MAAM,CAAC,QAAQ,OAAO,OAAO,QAAQ,QAAQ,GAChD,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO;IAAQ,UAAU;GAAE,CAAC;GAGjE,OAAO;EACT;EAEA,KAAK,gBACH,OAAO,CAAC;GAAC,IAAI;GAAgB,KAAK,OAAO;GAAK,KAAK,OAAO;EAAG,CAAC;EAEhE,KAAK,cACH,OAAO,CAAC;GAAC,IAAI;GAAc,OAAO,OAAO;EAAE,CAAC;EAE9C,KAAK,cAAc;GACjB,MAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;GACjD,MAAM,MAAkB,CAAC;GACzB,IAAI,eACF,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO,UAAU;GAAa,CAAC;GAElE,IAAI,UACF,IAAI,KAAK;IACP,IAAI;IACJ;IACA,QAAQ,OAAO;IACf;GACF,CAAC;GAEH,OAAO;EACT;EAEA,KAAK,iBAAiB;GAGpB,MAAM,EAAC,OAAO,KAAK,KAAK,YAAW;GACnC,OAAO,IAAI,SAAS,QAAQ,OACxB,CAAC,IACD,CAAC;IAAC,IAAI;IAAiB;IAAO,KAAK,IAAI;IAAM,KAAK,QAAQ;GAAI,CAAC;EACrE;EAEA,KAAK,eACH,OAAO,CAAC;GAAC,IAAI;GAAe,OAAO,OAAO;GAAO,QAAQ,OAAO;EAAM,CAAC;EAEzE,KAAK,sBAAsB;GACzB,MAAM,EAAC,UAAU,YAAW;GAG5B,MAAM,UAAU,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;GACvD,OAAO,QAAQ,WAAW,IACtB,CAAC,IACD,CACE;IACE,IAAI;IACJ,OAAO;KAAC,QAAQ,SAAS;KAAQ,MAAM,SAAS;IAAI;IACpD,SAAS;GACX,CACF;EACN;EAEA,KAAK;EACL,KAAK,cACH,OAAO,CAAC;EAEV,SAIE,YAAY,MAAM;CACtB;AACF;;;;;;;;;;;;;;AAeA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAKA,kBAAkB,GAAG,QAAgB;qBACzB,gCAAgC;;;;KAIhD;EACD,KAAKC,kBAAkB,GAAG,QAAgB;qBACzB,6BAA6B;;;;KAI7C;EACD,KAAKC,uBAAuB,GAAG,QAAgB;gBACnC,gCAAgC;;KAE3C;EACD,KAAKC,uBAAuB,GAAG,QAAgB;gBACnC,6BAA6B;;KAExC;EACD,KAAKC,uBAAuB,GAAG,QAAgB;qBAC9B,gCAAgC;;KAEhD;EACD,KAAKC,uBAAuB,GAAG,QAAgB;qBAC9B,6BAA6B;;KAE7C;EACD,KAAKC,wBAAwB,GAAG,QAAgB;gBACpC,6BAA6B;;KAExC;EACD,KAAKC,wBAAwB,GAAG,QAAgB;qBAC/B,6BAA6B;;KAE7C;CACH;;CAGA,MAAM,QAAkC;EACtC,MAAM,MAAM,UAAU,MAAM;EAC5B,KAAK,MAAM,MAAM,KACf,KAAKC,KAAK,EAAE;EAEd,OAAO;CACT;CAEA,KAAK,IAAoB;EACvB,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,KAAKR,gBAAgB,IACnB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,KAAKC,gBAAgB,IACnB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,QACH,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,KAAKC,qBAAqB,IACxB,GAAG,IAAI,QACP,GAAG,IAAI,MACP,GAAG,IAAI,QACP,GAAG,IAAI,IACT;IACA,KAAKC,qBAAqB,IACxB,GAAG,IAAI,QACP,GAAG,IAAI,MACP,GAAG,IAAI,QACP,GAAG,IAAI,IACT;IACA;GAEF,KAAK;IACH,KAAKC,qBAAqB,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IAC5D,KAAKC,qBAAqB,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IAC5D;GAEF,KAAK;IACH,KAAKC,sBAAsB,IACzB,GAAG,KACH,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,GACL;IACA;GAEF,KAAK;IACH,KAAKC,sBAAsB,IACzB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,MACL;IACA;GAEF,KAAK;IAIH,KAAK,MAAM,UAAU,GAAG,SACtB,KAAKA,sBAAsB,IACzB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,MACF;IAEF;GAEF,SACE,YAAY,EAAE;EAClB;CACF;AACF;AAEA,IAAM,YAAY,QAAgB,UAChC,KAAK,UAAU,CAAC,QAAQ,KAAK,CAAC;AAEhC,IAAM,aAAa,QAAgB,OAAe,WAChD,KAAK,UAAU;CAAC;CAAQ;CAAO;AAAM,CAAC;AAExC,IAAM,OAAO,GAAW,MAAe,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI;;;;;;;;;;;;;;;;AAiBhE,SAAgB,YACd,SACA,SACW;CACX,MAAM,WAAW,IAAI,IACnB,QAAQ,cAAc,KAAI,MAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CACjE;CACA,MAAM,cAAc,IAAI,IACtB,QAAQ,YAAY,KAAI,MAAK,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,CAC1E;CACA,MAAM,cAAc,EAAC,QAAQ,QAAmB,WAC9C,YAAY,OAAO,UAAU,QAAQ,MAAM,MAAM,CAAC;CAEpD,KAAK,MAAM,UAAU,SACnB,KAAK,MAAM,MAAM,UAAU,MAAM,GAC/B,QAAQ,GAAG,IAAX;EACE,KAAK;GACH,SAAS,IAAI,SAAS,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI,GAAG;IACrD,QAAQ,GAAG,MAAM;IACjB,OAAO,GAAG,MAAM;IAChB,UAAU,GAAG;GACf,CAAC;GACD;EAEF,KAAK;GACH,YAAY,IACV,UAAU,GAAG,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,MAAM,GACnD;IACE,QAAQ,GAAG,MAAM;IACjB,OAAO,GAAG,MAAM;IAChB,QAAQ,GAAG;IACX,UAAU,GAAG;GACf,CACF;GACA;EAEF,KAAK,gBAAgB;GACnB,MAAM,EAAC,KAAK,KAAK,YAAW;GAC5B,MAAM,QAAQ,SAAS,IAAI,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC;GACzD,IAAI,OAAO;IACT,SAAS,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC;IAC9C,SAAS,IAAI,SAAS,QAAQ,QAAQ,QAAQ,IAAI,GAAG;KACnD,GAAG;KACH,QAAQ,QAAQ;KAChB,OAAO,QAAQ;IACjB,CAAC;GACH;GAGA,KAAK,MAAM,CAAC,KAAK,WAAW,CAAC,GAAG,WAAW,GACzC,IAAI,OAAO,WAAW,IAAI,UAAU,OAAO,UAAU,IAAI,MAAM;IAC7D,YAAY,OAAO,GAAG;IACtB,YAAY,IACV,UAAU,QAAQ,QAAQ,QAAQ,MAAM,OAAO,MAAM,GACrD;KAAC,GAAG;KAAQ,QAAQ,QAAQ;KAAQ,OAAO,QAAQ;IAAI,CACzD;GACF;GAEF;EACF;EAEA,KAAK,cAAc;GACjB,MAAM,EAAC,QAAQ,SAAQ,GAAG;GAC1B,SAAS,OAAO,SAAS,QAAQ,IAAI,CAAC;GACtC,KAAK,MAAM,CAAC,KAAK,WAAW,CAAC,GAAG,WAAW,GACzC,IAAI,OAAO,WAAW,UAAU,OAAO,UAAU,MAC/C,YAAY,OAAO,GAAG;GAG1B;EACF;EAEA,KAAK,iBAAiB;GACpB,MAAM,EAAC,QAAQ,SAAQ,GAAG;GAC1B,MAAM,QAAQ,YAAY,IAAI,UAAU,QAAQ,MAAM,GAAG,GAAG,CAAC;GAC7D,IAAI,OAAO;IACT,YAAY,OAAO,UAAU,QAAQ,MAAM,GAAG,GAAG,CAAC;IAClD,YAAY,IAAI,UAAU,QAAQ,MAAM,GAAG,GAAG,GAAG;KAC/C,GAAG;KACH,QAAQ,GAAG;IACb,CAAC;GACH;GACA;EACF;EAEA,KAAK;GACH,WAAW,GAAG,OAAO,GAAG,MAAM;GAC9B;EAEF,KAAK;GACH,KAAK,MAAM,UAAU,GAAG,SACtB,WAAW,GAAG,OAAO,MAAM;GAE7B;EAEF,SACE,YAAY,EAAE;CAClB;CAIJ,OAAO;EACL,eAAe,CAAC,GAAG,SAAS,OAAO,CAAC,EAAE,UACnC,GAAG,MAAM,IAAI,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,EAAE,OAAO,EAAE,KAAK,CAC3D;EACA,aAAa,CAAC,GAAG,YAAY,OAAO,CAAC,EAAE,UACpC,GAAG,MACF,IAAI,EAAE,QAAQ,EAAE,MAAM,KACtB,IAAI,EAAE,OAAO,EAAE,KAAK,KACpB,IAAI,EAAE,QAAQ,EAAE,MAAM,CAC1B;CACF;AACF;AAEA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;AAM5D,SAAgB,qBAAqB,SAAuC;CAC1E,MAAM,WAAW,IAAI,IACnB,QAAQ,cAAc,KAAI,MAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,GAAG,EAAE,QAAQ,CAAC,CAC1E;CACA,MAAM,WAA8B,CAAC;CACrC,IAAI;CACJ,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,cAAa,QAAQ,aAAa;EACnE,IAAI,MAAM,MAAM,WAAW,UAAU,KAAK,MAAM,SAAS,OAAO;GAC9D,OAAO;IACL,OAAO;KACL;KACA,MAAM;KAGN,UAAU,SAAS,IAAI,SAAS,QAAQ,KAAK,CAAC,KAAK;IACrD;IACA,SAAS,CAAC;GACZ;GACA,SAAS,KAAK,IAAI;EACpB;EACA,KAAK,QAAQ,UAAU;CACzB;CACA,OAAO,MAAQ,UAAU,sBAAsB;AACjD;;;;;;AAOA,SAAgB,YAAY,IAAyB;CA4BnD,OAAO;EAAC,eA3Bc,GACnB,QAAgB;;gBAEL,gCAAgC;;KAE3C,EACA,IAAuD,EACvD,KAAK,EAAC,QAAQ,OAAO,gBAAe;GACnC;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAgBM;EAAe,aAdH,GACjB,QAAgB;;gBAEL,6BAA6B;;KAExC,EACA,IAAuE,EACvE,KAAK,EAAC,QAAQ,OAAO,QAAQ,gBAAe;GAC3C;GACA;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAEqB;CAAW;AACpC;;;;;;;;;;AAWA,SAAgB,eAAe,IAAc,SAA0B;CACrE,GAAG,QAAgB,gBAAgB,gCAAgC,EAAE,EAAE,IAAI;CAC3E,GAAG,QAAgB,gBAAgB,6BAA6B,EAAE,EAAE,IAAI;CAExE,IAAI,QAAQ,cAAc,SAAS,GAAG;EACpC,MAAM,SAAS,GAAG,QAAgB;qBACjB,gCAAgC;;KAEhD;EACD,KAAK,MAAM,EAAC,QAAQ,OAAO,cAAa,QAAQ,eAC9C,OAAO,IAAI,QAAQ,OAAO,WAAW,UAAU,QAAQ,CAAC;CAE5D;CACA,IAAI,QAAQ,YAAY,SAAS,GAAG;EAClC,MAAM,SAAS,GAAG,QAAgB;qBACjB,6BAA6B;;KAE7C;EACD,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,cAAa,QAAQ,aACtD,OAAO,IAAI,QAAQ,OAAO,QAAQ,WAAW,UAAU,QAAQ,CAAC;CAEpE;AACF;AAQA,SAAgB,oBAAoB,IAA+B;CACjE,OAAO,GACJ,QAAgB;;iCAEY,gCAAgC;;iCAEhC,6BAA6B;;KAEzD,EACA,IAAqB;AAC1B"}
+{"version":3,"file":"change-log-cookies.js","names":["#upsertMetadata","#upsertBackfill","#renameMetadataTable","#renameBackfillTable","#deleteMetadataTable","#deleteBackfillTable","#renameBackfillColumn","#deleteBackfillColumn","#run"],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-cookies.ts"],"sourcesContent":["/**\n * The change log's *second* job.\n *\n * A change log is a buffer **and** a cookie jar. The buffer catches a\n * reconnecting subscriber up to the head; the cookie jar holds the backfill\n * progress state that the change-streamer hands the change-source on every\n * `startStream`, so that the change source never has to persist backfill state\n * upstream. See `change-log-db.ts` for the buffer.\n *\n * There are two cookies, mirroring the two `cdc` tables the Postgres change log\n * has always carried (`change-streamer/schema/tables.ts`):\n *\n * - **table metadata**, the most current {@link TableMetadata} of a table, which\n *   a {@link BackfillRequest} has to carry; and\n * - **backfilling columns**, the opaque {@link BackfillID} of every column whose\n *   backfill has not completed.\n *\n * Both are a *fold over the change stream*: the cookie set is only meaningful\n * paired with the watermark it was folded up to. {@link cookieOps} is that fold,\n * expressed transport-free so that the Postgres change log and the SQLite change\n * log cannot drift from each other — they must agree on all eight transitions\n * forever, and the failure mode if they disagree is a backfill that is silently\n * never re-requested. The two interpreters are {@link ChangeLogCookieWriter}\n * here and `Storer.#trackBackfillMetadata()` in\n * `change-streamer/storer.ts`.\n *\n * The tables live in the change-log database rather than in the replica: the\n * replicator is a subscriber *downstream* of the change-streamer's write loop,\n * so its state version trails the log's head, and pairing its cookies with the\n * log's resume watermark would lose every backfill that completed in between.\n * The replica's own copies (`_zero.tableMetadata`, `_zero.column_metadata`) are\n * the *seed* source; these are the *working* set.\n */\n\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Database, Statement} from '../../../../zqlite/src/db.ts';\nimport type {\n  BackfillID,\n  Identifier,\n  SchemaChange,\n  TableMetadata,\n} from '../change-source/protocol/current/data.ts';\nimport {\n  backfillRequestSchema,\n  type BackfillRequest,\n} from '../change-source/protocol/current/upstream.ts';\n\nexport const CHANGE_LOG_TABLE_METADATA_TABLE = '_zero.changeLogTableMetadata';\nexport const CHANGE_LOG_BACKFILLING_TABLE = '_zero.changeLogBackfilling';\n\n// The names follow the file's existing convention, beside\n// \"_zero.changeLogStream\" and \"_zero.changeLogMeta\".\n//\n// `metadata` and `backfill` hold the JSON that the `cdc` tables hold as JSONB.\n// SQLite has no JSONB, and nothing here queries into the documents: they are\n// stored to be handed back to the change source verbatim.\nexport const CREATE_CHANGE_LOG_COOKIE_SCHEMA = /*sql*/ `\n  CREATE TABLE \"${CHANGE_LOG_TABLE_METADATA_TABLE}\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"metadata\" TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n\n  CREATE TABLE \"${CHANGE_LOG_BACKFILLING_TABLE}\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n`;\n\nexport const DROP_CHANGE_LOG_COOKIE_TABLES = /*sql*/ `\n  DROP TABLE IF EXISTS \"${CHANGE_LOG_TABLE_METADATA_TABLE}\";\n  DROP TABLE IF EXISTS \"${CHANGE_LOG_BACKFILLING_TABLE}\";\n`;\n\nexport type TableMetadataCookie = {\n  readonly schema: string;\n  readonly table: string;\n  readonly metadata: TableMetadata;\n};\n\nexport type BackfillCookie = {\n  readonly schema: string;\n  readonly table: string;\n  readonly column: string;\n  readonly backfill: BackfillID;\n};\n\n/**\n * The whole cookie set, as of the log's head. Both lists are ordered by their\n * primary key so that two stores' sets can be compared directly.\n */\nexport type CookieSet = {\n  readonly tableMetadata: readonly TableMetadataCookie[];\n  readonly backfilling: readonly BackfillCookie[];\n};\n\nexport const EMPTY_COOKIE_SET: CookieSet = {\n  tableMetadata: [],\n  backfilling: [],\n};\n\n/**\n * A transport-free description of a cookie mutation. This is the whole of what\n * a {@link SchemaChange} means to the cookie jar; everything else about the\n * change belongs to the buffer.\n */\nexport type CookieOp =\n  | {op: 'upsert-metadata'; table: Identifier; metadata: TableMetadata}\n  | {\n      op: 'upsert-backfill';\n      table: Identifier;\n      column: string;\n      backfill: BackfillID;\n    }\n  | {op: 'rename-table'; old: Identifier; new: Identifier}\n  | {op: 'drop-table'; table: Identifier}\n  | {op: 'rename-column'; table: Identifier; old: string; new: string}\n  | {op: 'drop-column'; table: Identifier; column: string}\n  | {op: 'complete-backfill'; table: Identifier; columns: readonly string[]};\n\nexport type CookieOpTag = CookieOp['op'];\n\n/**\n * The fold, and the only place either store decides what a schema change means\n * for the cookie set. Both interpreters are mechanical translations of what this\n * returns, in the order it returns them.\n *\n * Changes that carry no cookie state — index changes, and the `create-table` /\n * `add-column` variants whose optional `metadata` and `backfill` fields are\n * absent — produce no ops.\n */\nexport function cookieOps(change: SchemaChange): CookieOp[] {\n  switch (change.tag) {\n    case 'update-table-metadata':\n      return [\n        {op: 'upsert-metadata', table: change.table, metadata: change.new},\n      ];\n\n    case 'create-table': {\n      const {spec, metadata, backfill} = change;\n      const table = {schema: spec.schema, name: spec.name};\n      const ops: CookieOp[] = [];\n      if (metadata) {\n        ops.push({op: 'upsert-metadata', table, metadata});\n      }\n      if (backfill) {\n        for (const [column, id] of Object.entries(backfill)) {\n          ops.push({op: 'upsert-backfill', table, column, backfill: id});\n        }\n      }\n      return ops;\n    }\n\n    case 'rename-table':\n      return [{op: 'rename-table', old: change.old, new: change.new}];\n\n    case 'drop-table':\n      return [{op: 'drop-table', table: change.id}];\n\n    case 'add-column': {\n      const {table, tableMetadata, column, backfill} = change;\n      const ops: CookieOp[] = [];\n      if (tableMetadata) {\n        ops.push({op: 'upsert-metadata', table, metadata: tableMetadata});\n      }\n      if (backfill) {\n        ops.push({\n          op: 'upsert-backfill',\n          table,\n          column: column.name,\n          backfill,\n        });\n      }\n      return ops;\n    }\n\n    case 'update-column': {\n      // Only a rename moves the cookie; the rest of a column update does not\n      // affect whether or how it is being backfilled.\n      const {table, old, new: updated} = change;\n      return old.name === updated.name\n        ? []\n        : [{op: 'rename-column', table, old: old.name, new: updated.name}];\n    }\n\n    case 'drop-column':\n      return [{op: 'drop-column', table: change.table, column: change.column}];\n\n    case 'backfill-completed': {\n      const {relation, columns} = change;\n      // The rowKey columns are excluded from `columns` but are backfilled with\n      // them, so both are cleared.\n      const cleared = [...relation.rowKey.columns, ...columns];\n      return cleared.length === 0\n        ? []\n        : [\n            {\n              op: 'complete-backfill',\n              table: {schema: relation.schema, name: relation.name},\n              columns: cleared,\n            },\n          ];\n    }\n\n    case 'create-index':\n    case 'drop-index':\n      return [];\n\n    default:\n      // Unreachable: `isSchemaChange()` gates on the same tag set this switch\n      // covers. It is exhaustive so that a new schema-change tag is a compile\n      // error here rather than a silently dropped backfill.\n      unreachable(change);\n  }\n}\n\n/**\n * The SQLite interpreter of {@link cookieOps}, over prepared statements on the\n * change-log database.\n *\n * The statements run inside the log's transaction, in stream order, at the point\n * the schema change is appended (see `ChangeLogStreamWriter`). That is what\n * makes the cookie set atomic with the head — a crash cannot leave cookies ahead\n * of or behind the rows they were folded from — and what makes an aborted\n * transaction discard its cookies with its rows, for free.\n *\n * Statements are prepared against the tables as they exist now, so an instance\n * does not survive a reconcile that drops and recreates them.\n */\nexport class ChangeLogCookieWriter {\n  readonly #upsertMetadata: Statement;\n  readonly #upsertBackfill: Statement;\n  readonly #renameMetadataTable: Statement;\n  readonly #renameBackfillTable: Statement;\n  readonly #deleteMetadataTable: Statement;\n  readonly #deleteBackfillTable: Statement;\n  readonly #renameBackfillColumn: Statement;\n  readonly #deleteBackfillColumn: Statement;\n\n  constructor(db: Database) {\n    this.#upsertMetadata = db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n        (\"schema\", \"table\", \"metadata\") VALUES (?, ?, ?)\n        ON CONFLICT (\"schema\", \"table\")\n        DO UPDATE SET \"metadata\" = excluded.\"metadata\"\n    `);\n    this.#upsertBackfill = db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n        (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n        ON CONFLICT (\"schema\", \"table\", \"column\")\n        DO UPDATE SET \"backfill\" = excluded.\"backfill\"\n    `);\n    this.#renameMetadataTable = db.prepare(/*sql*/ `\n      UPDATE \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n        SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n    `);\n    this.#renameBackfillTable = db.prepare(/*sql*/ `\n      UPDATE \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n        SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n    `);\n    this.#deleteMetadataTable = db.prepare(/*sql*/ `\n      DELETE FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ?\n    `);\n    this.#deleteBackfillTable = db.prepare(/*sql*/ `\n      DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ?\n    `);\n    this.#renameBackfillColumn = db.prepare(/*sql*/ `\n      UPDATE \"${CHANGE_LOG_BACKFILLING_TABLE}\" SET \"column\" = ?\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `);\n    this.#deleteBackfillColumn = db.prepare(/*sql*/ `\n      DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `);\n  }\n\n  /** Applies the change's cookie ops, returning the ops that were applied. */\n  apply(change: SchemaChange): CookieOp[] {\n    const ops = cookieOps(change);\n    for (const op of ops) {\n      this.#run(op);\n    }\n    return ops;\n  }\n\n  #run(op: CookieOp): void {\n    switch (op.op) {\n      case 'upsert-metadata':\n        this.#upsertMetadata.run(\n          op.table.schema,\n          op.table.name,\n          BigIntJSON.stringify(op.metadata),\n        );\n        break;\n\n      case 'upsert-backfill':\n        this.#upsertBackfill.run(\n          op.table.schema,\n          op.table.name,\n          op.column,\n          BigIntJSON.stringify(op.backfill),\n        );\n        break;\n\n      case 'rename-table':\n        this.#renameMetadataTable.run(\n          op.new.schema,\n          op.new.name,\n          op.old.schema,\n          op.old.name,\n        );\n        this.#renameBackfillTable.run(\n          op.new.schema,\n          op.new.name,\n          op.old.schema,\n          op.old.name,\n        );\n        break;\n\n      case 'drop-table':\n        this.#deleteMetadataTable.run(op.table.schema, op.table.name);\n        this.#deleteBackfillTable.run(op.table.schema, op.table.name);\n        break;\n\n      case 'rename-column':\n        this.#renameBackfillColumn.run(\n          op.new,\n          op.table.schema,\n          op.table.name,\n          op.old,\n        );\n        break;\n\n      case 'drop-column':\n        this.#deleteBackfillColumn.run(\n          op.table.schema,\n          op.table.name,\n          op.column,\n        );\n        break;\n\n      case 'complete-backfill':\n        // A per-column delete rather than an `IN` list, which cannot be a\n        // single prepared statement across arities. The columns of one\n        // completed backfill are few, and this runs only on schema changes.\n        for (const column of op.columns) {\n          this.#deleteBackfillColumn.run(\n            op.table.schema,\n            op.table.name,\n            column,\n          );\n        }\n        break;\n\n      default:\n        unreachable(op);\n    }\n  }\n}\n\nconst tableKey = (schema: string, table: string) =>\n  JSON.stringify([schema, table]);\n\nconst columnKey = (schema: string, table: string, column: string) =>\n  JSON.stringify([schema, table, column]);\n\nconst cmp = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0);\n\n/**\n * The in-memory interpreter of {@link cookieOps}: the third one, and the only\n * one that is not a store.\n *\n * A replica-derived cookie set is a snapshot at the replica's state version,\n * which trails the change log's head, so it cannot be compared with a\n * Postgres-derived set directly. This carries it forward over the schema changes\n * the log holds in between, which turns \"the two stores disagree, but they are\n * also at different positions\" into an exact equality — and makes a\n * disagreement between any two of the three interpreters visible as an\n * inequality that neither store would have noticed on its own.\n *\n * `changes` must be the log's schema changes over `(from, to]` in stream order.\n * Anything else produces a set that belongs to no watermark.\n */\nexport function foldCookies(\n  cookies: CookieSet,\n  changes: Iterable<SchemaChange>,\n): CookieSet {\n  const metadata = new Map<string, TableMetadataCookie>(\n    cookies.tableMetadata.map(c => [tableKey(c.schema, c.table), c]),\n  );\n  const backfilling = new Map<string, BackfillCookie>(\n    cookies.backfilling.map(c => [columnKey(c.schema, c.table, c.column), c]),\n  );\n  const dropColumn = ({schema, name}: Identifier, column: string) =>\n    backfilling.delete(columnKey(schema, name, column));\n\n  for (const change of changes) {\n    for (const op of cookieOps(change)) {\n      switch (op.op) {\n        case 'upsert-metadata':\n          metadata.set(tableKey(op.table.schema, op.table.name), {\n            schema: op.table.schema,\n            table: op.table.name,\n            metadata: op.metadata,\n          });\n          break;\n\n        case 'upsert-backfill':\n          backfilling.set(\n            columnKey(op.table.schema, op.table.name, op.column),\n            {\n              schema: op.table.schema,\n              table: op.table.name,\n              column: op.column,\n              backfill: op.backfill,\n            },\n          );\n          break;\n\n        case 'rename-table': {\n          const {old, new: renamed} = op;\n          const moved = metadata.get(tableKey(old.schema, old.name));\n          if (moved) {\n            metadata.delete(tableKey(old.schema, old.name));\n            metadata.set(tableKey(renamed.schema, renamed.name), {\n              ...moved,\n              schema: renamed.schema,\n              table: renamed.name,\n            });\n          }\n          // Snapshotted before mutating, since the rename both deletes and\n          // inserts into the map being scanned.\n          for (const [key, cookie] of [...backfilling]) {\n            if (cookie.schema === old.schema && cookie.table === old.name) {\n              backfilling.delete(key);\n              backfilling.set(\n                columnKey(renamed.schema, renamed.name, cookie.column),\n                {...cookie, schema: renamed.schema, table: renamed.name},\n              );\n            }\n          }\n          break;\n        }\n\n        case 'drop-table': {\n          const {schema, name} = op.table;\n          metadata.delete(tableKey(schema, name));\n          for (const [key, cookie] of [...backfilling]) {\n            if (cookie.schema === schema && cookie.table === name) {\n              backfilling.delete(key);\n            }\n          }\n          break;\n        }\n\n        case 'rename-column': {\n          const {schema, name} = op.table;\n          const moved = backfilling.get(columnKey(schema, name, op.old));\n          if (moved) {\n            backfilling.delete(columnKey(schema, name, op.old));\n            backfilling.set(columnKey(schema, name, op.new), {\n              ...moved,\n              column: op.new,\n            });\n          }\n          break;\n        }\n\n        case 'drop-column':\n          dropColumn(op.table, op.column);\n          break;\n\n        case 'complete-backfill':\n          for (const column of op.columns) {\n            dropColumn(op.table, column);\n          }\n          break;\n\n        default:\n          unreachable(op);\n      }\n    }\n  }\n\n  return {\n    tableMetadata: [...metadata.values()].toSorted(\n      (a, b) => cmp(a.schema, b.schema) || cmp(a.table, b.table),\n    ),\n    backfilling: [...backfilling.values()].toSorted(\n      (a, b) =>\n        cmp(a.schema, b.schema) ||\n        cmp(a.table, b.table) ||\n        cmp(a.column, b.column),\n    ),\n  };\n}\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\n/**\n * Returns one {@link BackfillRequest} for each table with an active backfill.\n * Each request contains the stored table metadata, if available.\n */\nexport function backfillRequestsFrom(cookies: CookieSet): BackfillRequest[] {\n  const metadata = new Map(\n    cookies.tableMetadata.map(c => [tableKey(c.schema, c.table), c.metadata]),\n  );\n  const requests: BackfillRequest[] = [];\n  let curr: BackfillRequest | undefined;\n  for (const {schema, table, column, backfill} of cookies.backfilling) {\n    if (curr?.table.schema !== schema || curr.table.name !== table) {\n      curr = {\n        table: {\n          schema,\n          name: table,\n          // Use `null` when a backfilling table has no metadata. This matches\n          // the result of the Postgres left join.\n          metadata: metadata.get(tableKey(schema, table)) ?? null,\n        },\n        columns: {},\n      };\n      requests.push(curr);\n    }\n    curr.columns[column] = {id: backfill};\n  }\n  return v.parse(requests, backfillRequestsSchema);\n}\n\n/**\n * Reads the whole cookie set, ordered by primary key. This is what a\n * change-streamer would hand `startStream`, and what the Postgres-derived and\n * replica-derived sets are compared against.\n */\nexport function readCookies(db: Database): CookieSet {\n  const tableMetadata = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"metadata\"\n        FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n        ORDER BY \"schema\", \"table\"\n    `)\n    .all<{schema: string; table: string; metadata: string}>()\n    .map(({schema, table, metadata}) => ({\n      schema,\n      table,\n      metadata: BigIntJSON.parse(metadata) as TableMetadata,\n    }));\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"column\", \"backfill\"\n        FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n        ORDER BY \"schema\", \"table\", \"column\"\n    `)\n    .all<{schema: string; table: string; column: string; backfill: string}>()\n    .map(({schema, table, column, backfill}) => ({\n      schema,\n      table,\n      column,\n      backfill: BigIntJSON.parse(backfill) as BackfillID,\n    }));\n\n  return {tableMetadata, backfilling};\n}\n\n/**\n * Replaces the cookie set wholesale.\n *\n * The cookies are unversioned point-in-time state, so a reconciliation that\n * moves the head invalidates them: truncating the stream above `W` does not roll\n * the cookie set back to `W`, and the truncated transactions can contain\n * `create-table`, `add-column`, and `backfill-completed`. The caller must run\n * this in the same transaction as the reconciliation that invalidated them.\n */\nexport function replaceCookies(db: Database, cookies: CookieSet): void {\n  db.prepare(/*sql*/ `DELETE FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"`).run();\n  db.prepare(/*sql*/ `DELETE FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\"`).run();\n\n  if (cookies.tableMetadata.length > 0) {\n    const insert = db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_TABLE_METADATA_TABLE}\"\n        (\"schema\", \"table\", \"metadata\") VALUES (?, ?, ?)\n    `);\n    for (const {schema, table, metadata} of cookies.tableMetadata) {\n      insert.run(schema, table, BigIntJSON.stringify(metadata));\n    }\n  }\n  if (cookies.backfilling.length > 0) {\n    const insert = db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_BACKFILLING_TABLE}\"\n        (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n    `);\n    for (const {schema, table, column, backfill} of cookies.backfilling) {\n      insert.run(schema, table, column, BigIntJSON.stringify(backfill));\n    }\n  }\n}\n\n/** Retained cookie rows, by table. Both are normally zero. */\nexport type CookieRowCounts = {\n  readonly tableMetadata: number;\n  readonly backfilling: number;\n};\n\nexport function readCookieRowCounts(db: Database): CookieRowCounts {\n  return db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT count(*) FROM \"${CHANGE_LOG_TABLE_METADATA_TABLE}\")\n          AS \"tableMetadata\",\n        (SELECT count(*) FROM \"${CHANGE_LOG_BACKFILLING_TABLE}\")\n          AS \"backfilling\"\n    `)\n    .get<CookieRowCounts>();\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,IAAa,kCAAkC;AAC/C,IAAa,+BAA+B;AAQ5C,IAAa,kCAA0C;kBACrC,gCAAgC;;;;;;;kBAOhC,6BAA6B;;;;;;;;AAS/C,IAAa,gCAAwC;0BAC3B,gCAAgC;0BAChC,6BAA6B;;;;;;;;;;;AA4DvD,SAAgB,UAAU,QAAkC;CAC1D,QAAQ,OAAO,KAAf;EACE,KAAK,yBACH,OAAO,CACL;GAAC,IAAI;GAAmB,OAAO,OAAO;GAAO,UAAU,OAAO;EAAG,CACnE;EAEF,KAAK,gBAAgB;GACnB,MAAM,EAAC,MAAM,UAAU,aAAY;GACnC,MAAM,QAAQ;IAAC,QAAQ,KAAK;IAAQ,MAAM,KAAK;GAAI;GACnD,MAAM,MAAkB,CAAC;GACzB,IAAI,UACF,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO;GAAQ,CAAC;GAEnD,IAAI,UACF,KAAK,MAAM,CAAC,QAAQ,OAAO,OAAO,QAAQ,QAAQ,GAChD,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO;IAAQ,UAAU;GAAE,CAAC;GAGjE,OAAO;EACT;EAEA,KAAK,gBACH,OAAO,CAAC;GAAC,IAAI;GAAgB,KAAK,OAAO;GAAK,KAAK,OAAO;EAAG,CAAC;EAEhE,KAAK,cACH,OAAO,CAAC;GAAC,IAAI;GAAc,OAAO,OAAO;EAAE,CAAC;EAE9C,KAAK,cAAc;GACjB,MAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;GACjD,MAAM,MAAkB,CAAC;GACzB,IAAI,eACF,IAAI,KAAK;IAAC,IAAI;IAAmB;IAAO,UAAU;GAAa,CAAC;GAElE,IAAI,UACF,IAAI,KAAK;IACP,IAAI;IACJ;IACA,QAAQ,OAAO;IACf;GACF,CAAC;GAEH,OAAO;EACT;EAEA,KAAK,iBAAiB;GAGpB,MAAM,EAAC,OAAO,KAAK,KAAK,YAAW;GACnC,OAAO,IAAI,SAAS,QAAQ,OACxB,CAAC,IACD,CAAC;IAAC,IAAI;IAAiB;IAAO,KAAK,IAAI;IAAM,KAAK,QAAQ;GAAI,CAAC;EACrE;EAEA,KAAK,eACH,OAAO,CAAC;GAAC,IAAI;GAAe,OAAO,OAAO;GAAO,QAAQ,OAAO;EAAM,CAAC;EAEzE,KAAK,sBAAsB;GACzB,MAAM,EAAC,UAAU,YAAW;GAG5B,MAAM,UAAU,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;GACvD,OAAO,QAAQ,WAAW,IACtB,CAAC,IACD,CACE;IACE,IAAI;IACJ,OAAO;KAAC,QAAQ,SAAS;KAAQ,MAAM,SAAS;IAAI;IACpD,SAAS;GACX,CACF;EACN;EAEA,KAAK;EACL,KAAK,cACH,OAAO,CAAC;EAEV,SAIE,YAAY,MAAM;CACtB;AACF;;;;;;;;;;;;;;AAeA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAKA,kBAAkB,GAAG,QAAgB;qBACzB,gCAAgC;;;;KAIhD;EACD,KAAKC,kBAAkB,GAAG,QAAgB;qBACzB,6BAA6B;;;;KAI7C;EACD,KAAKC,uBAAuB,GAAG,QAAgB;gBACnC,gCAAgC;;KAE3C;EACD,KAAKC,uBAAuB,GAAG,QAAgB;gBACnC,6BAA6B;;KAExC;EACD,KAAKC,uBAAuB,GAAG,QAAgB;qBAC9B,gCAAgC;;KAEhD;EACD,KAAKC,uBAAuB,GAAG,QAAgB;qBAC9B,6BAA6B;;KAE7C;EACD,KAAKC,wBAAwB,GAAG,QAAgB;gBACpC,6BAA6B;;KAExC;EACD,KAAKC,wBAAwB,GAAG,QAAgB;qBAC/B,6BAA6B;;KAE7C;CACH;;CAGA,MAAM,QAAkC;EACtC,MAAM,MAAM,UAAU,MAAM;EAC5B,KAAK,MAAM,MAAM,KACf,KAAKC,KAAK,EAAE;EAEd,OAAO;CACT;CAEA,KAAK,IAAoB;EACvB,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,KAAKR,gBAAgB,IACnB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,KAAKC,gBAAgB,IACnB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,QACH,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,KAAKC,qBAAqB,IACxB,GAAG,IAAI,QACP,GAAG,IAAI,MACP,GAAG,IAAI,QACP,GAAG,IAAI,IACT;IACA,KAAKC,qBAAqB,IACxB,GAAG,IAAI,QACP,GAAG,IAAI,MACP,GAAG,IAAI,QACP,GAAG,IAAI,IACT;IACA;GAEF,KAAK;IACH,KAAKC,qBAAqB,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IAC5D,KAAKC,qBAAqB,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IAC5D;GAEF,KAAK;IACH,KAAKC,sBAAsB,IACzB,GAAG,KACH,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,GACL;IACA;GAEF,KAAK;IACH,KAAKC,sBAAsB,IACzB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,MACL;IACA;GAEF,KAAK;IAIH,KAAK,MAAM,UAAU,GAAG,SACtB,KAAKA,sBAAsB,IACzB,GAAG,MAAM,QACT,GAAG,MAAM,MACT,MACF;IAEF;GAEF,SACE,YAAY,EAAE;EAClB;CACF;AACF;AAEA,IAAM,YAAY,QAAgB,UAChC,KAAK,UAAU,CAAC,QAAQ,KAAK,CAAC;AAEhC,IAAM,aAAa,QAAgB,OAAe,WAChD,KAAK,UAAU;CAAC;CAAQ;CAAO;AAAM,CAAC;AAExC,IAAM,OAAO,GAAW,MAAe,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI;;;;;;;;;;;;;;;;AAiBhE,SAAgB,YACd,SACA,SACW;CACX,MAAM,WAAW,IAAI,IACnB,QAAQ,cAAc,KAAI,MAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CACjE;CACA,MAAM,cAAc,IAAI,IACtB,QAAQ,YAAY,KAAI,MAAK,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,CAC1E;CACA,MAAM,cAAc,EAAC,QAAQ,QAAmB,WAC9C,YAAY,OAAO,UAAU,QAAQ,MAAM,MAAM,CAAC;CAEpD,KAAK,MAAM,UAAU,SACnB,KAAK,MAAM,MAAM,UAAU,MAAM,GAC/B,QAAQ,GAAG,IAAX;EACE,KAAK;GACH,SAAS,IAAI,SAAS,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI,GAAG;IACrD,QAAQ,GAAG,MAAM;IACjB,OAAO,GAAG,MAAM;IAChB,UAAU,GAAG;GACf,CAAC;GACD;EAEF,KAAK;GACH,YAAY,IACV,UAAU,GAAG,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,MAAM,GACnD;IACE,QAAQ,GAAG,MAAM;IACjB,OAAO,GAAG,MAAM;IAChB,QAAQ,GAAG;IACX,UAAU,GAAG;GACf,CACF;GACA;EAEF,KAAK,gBAAgB;GACnB,MAAM,EAAC,KAAK,KAAK,YAAW;GAC5B,MAAM,QAAQ,SAAS,IAAI,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC;GACzD,IAAI,OAAO;IACT,SAAS,OAAO,SAAS,IAAI,QAAQ,IAAI,IAAI,CAAC;IAC9C,SAAS,IAAI,SAAS,QAAQ,QAAQ,QAAQ,IAAI,GAAG;KACnD,GAAG;KACH,QAAQ,QAAQ;KAChB,OAAO,QAAQ;IACjB,CAAC;GACH;GAGA,KAAK,MAAM,CAAC,KAAK,WAAW,CAAC,GAAG,WAAW,GACzC,IAAI,OAAO,WAAW,IAAI,UAAU,OAAO,UAAU,IAAI,MAAM;IAC7D,YAAY,OAAO,GAAG;IACtB,YAAY,IACV,UAAU,QAAQ,QAAQ,QAAQ,MAAM,OAAO,MAAM,GACrD;KAAC,GAAG;KAAQ,QAAQ,QAAQ;KAAQ,OAAO,QAAQ;IAAI,CACzD;GACF;GAEF;EACF;EAEA,KAAK,cAAc;GACjB,MAAM,EAAC,QAAQ,SAAQ,GAAG;GAC1B,SAAS,OAAO,SAAS,QAAQ,IAAI,CAAC;GACtC,KAAK,MAAM,CAAC,KAAK,WAAW,CAAC,GAAG,WAAW,GACzC,IAAI,OAAO,WAAW,UAAU,OAAO,UAAU,MAC/C,YAAY,OAAO,GAAG;GAG1B;EACF;EAEA,KAAK,iBAAiB;GACpB,MAAM,EAAC,QAAQ,SAAQ,GAAG;GAC1B,MAAM,QAAQ,YAAY,IAAI,UAAU,QAAQ,MAAM,GAAG,GAAG,CAAC;GAC7D,IAAI,OAAO;IACT,YAAY,OAAO,UAAU,QAAQ,MAAM,GAAG,GAAG,CAAC;IAClD,YAAY,IAAI,UAAU,QAAQ,MAAM,GAAG,GAAG,GAAG;KAC/C,GAAG;KACH,QAAQ,GAAG;IACb,CAAC;GACH;GACA;EACF;EAEA,KAAK;GACH,WAAW,GAAG,OAAO,GAAG,MAAM;GAC9B;EAEF,KAAK;GACH,KAAK,MAAM,UAAU,GAAG,SACtB,WAAW,GAAG,OAAO,MAAM;GAE7B;EAEF,SACE,YAAY,EAAE;CAClB;CAIJ,OAAO;EACL,eAAe,CAAC,GAAG,SAAS,OAAO,CAAC,EAAE,UACnC,GAAG,MAAM,IAAI,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,EAAE,OAAO,EAAE,KAAK,CAC3D;EACA,aAAa,CAAC,GAAG,YAAY,OAAO,CAAC,EAAE,UACpC,GAAG,MACF,IAAI,EAAE,QAAQ,EAAE,MAAM,KACtB,IAAI,EAAE,OAAO,EAAE,KAAK,KACpB,IAAI,EAAE,QAAQ,EAAE,MAAM,CAC1B;CACF;AACF;AAEA,IAAM,yBAAyB,eAAE,MAAM,qBAAqB;;;;;AAM5D,SAAgB,qBAAqB,SAAuC;CAC1E,MAAM,WAAW,IAAI,IACnB,QAAQ,cAAc,KAAI,MAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,GAAG,EAAE,QAAQ,CAAC,CAC1E;CACA,MAAM,WAA8B,CAAC;CACrC,IAAI;CACJ,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,cAAa,QAAQ,aAAa;EACnE,IAAI,MAAM,MAAM,WAAW,UAAU,KAAK,MAAM,SAAS,OAAO;GAC9D,OAAO;IACL,OAAO;KACL;KACA,MAAM;KAGN,UAAU,SAAS,IAAI,SAAS,QAAQ,KAAK,CAAC,KAAK;IACrD;IACA,SAAS,CAAC;GACZ;GACA,SAAS,KAAK,IAAI;EACpB;EACA,KAAK,QAAQ,UAAU,EAAC,IAAI,SAAQ;CACtC;CACA,OAAO,MAAQ,UAAU,sBAAsB;AACjD;;;;;;AAOA,SAAgB,YAAY,IAAyB;CA4BnD,OAAO;EAAC,eA3Bc,GACnB,QAAgB;;gBAEL,gCAAgC;;KAE3C,EACA,IAAuD,EACvD,KAAK,EAAC,QAAQ,OAAO,gBAAe;GACnC;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAgBM;EAAe,aAdH,GACjB,QAAgB;;gBAEL,6BAA6B;;KAExC,EACA,IAAuE,EACvE,KAAK,EAAC,QAAQ,OAAO,QAAQ,gBAAe;GAC3C;GACA;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAEqB;CAAW;AACpC;;;;;;;;;;AAWA,SAAgB,eAAe,IAAc,SAA0B;CACrE,GAAG,QAAgB,gBAAgB,gCAAgC,EAAE,EAAE,IAAI;CAC3E,GAAG,QAAgB,gBAAgB,6BAA6B,EAAE,EAAE,IAAI;CAExE,IAAI,QAAQ,cAAc,SAAS,GAAG;EACpC,MAAM,SAAS,GAAG,QAAgB;qBACjB,gCAAgC;;KAEhD;EACD,KAAK,MAAM,EAAC,QAAQ,OAAO,cAAa,QAAQ,eAC9C,OAAO,IAAI,QAAQ,OAAO,WAAW,UAAU,QAAQ,CAAC;CAE5D;CACA,IAAI,QAAQ,YAAY,SAAS,GAAG;EAClC,MAAM,SAAS,GAAG,QAAgB;qBACjB,6BAA6B;;KAE7C;EACD,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,cAAa,QAAQ,aACtD,OAAO,IAAI,QAAQ,OAAO,QAAQ,WAAW,UAAU,QAAQ,CAAC;CAEpE;AACF;AAQA,SAAgB,oBAAoB,IAA+B;CACjE,OAAO,GACJ,QAAgB;;iCAEY,gCAAgC;;iCAEhC,6BAA6B;;KAEzD,EACA,IAAqB;AAC1B"}

package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts

@@ -4,7 +4,8 @@ import { type CookieOpTag } from './change-log-cookies.ts';
 export type ChangeLogStreamTransactionStats = {
     readonly rows: number;
     readonly estimatedBytes: number;
-    readonly hash: string;
+    /** Present only for transactions selected for shadow-write validation. */
+    readonly hash?: string | undefined;
     /**
      * The cookie mutations this transaction's schema changes folded in, in stream
      * order. Empty for the overwhelming majority of transactions.
@@ -32,7 +33,7 @@ export declare class ChangeLogStreamInvariantError extends Error {
 export declare class ChangeLogStreamWriter {
     #private;
     constructor(db: StatementRunner);
-    begin(watermark: string, json: string): void;
+    begin(watermark: string, json: string, storedChange?: string, hash?: boolean): void;
     /**
      * Appends one data or schema change. The parsed message is taken rather than
      * just its tag because a schema change also mutates the cookie jar, and the
@@ -43,8 +44,8 @@ export declare class ChangeLogStreamWriter {
      * has to flush its buffered batch to get the same ordering
      * (`Storer.#processQueue()`); SQLite gets it from statement order.
      */
-    append(json: string, change: DataOrSchemaChange): void;
-    commit(watermark: string, json: string, writeTimeMs: number): ChangeLogStreamTransactionStats;
+    append(json: string, change: DataOrSchemaChange, storedChange?: string): void;
+    commit(watermark: string, json: string, writeTimeMs: number, storedChange?: string): ChangeLogStreamTransactionStats;
     /**
      * Discards the open transaction, if any. Safe to call when none is open — the
      * caller rolls back on an interrupted change stream, which can arrive between

package/out/zero-cache/src/services/replicator/change-log-stream-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-stream-writer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-log-stream-writer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAEL,KAAK,kBAAkB,EACxB,MAAM,2CAA2C,CAAC;AAGnD,OAAO,EAAwB,KAAK,WAAW,EAAC,MAAM,yBAAyB,CAAC;AAMhF,MAAM,MAAM,+BAA+B,GAAG;IAC5C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,QAAQ,CAAC,eAAe,EAAE,SAAS,WAAW,EAAE,CAAC;CAClD,CAAC;AAEF,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,SAAkB,IAAI,mCAAmC;CAC1D;AAWD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,qBAAqB;;IAYhC,YAAY,EAAE,EAAE,eAAe,EAc9B;IAED,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAiC3C;IAED;;;;;;;;;OASG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,GAAG,IAAI,CAwBrD;IAED,MAAM,CACJ,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,GAClB,+BAA+B,CA2CjC;IAED;;;;OAIG;IACH,QAAQ,IAAI,IAAI,CAMf;CA2BF"}
+{"version":3,"file":"change-log-stream-writer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-log-stream-writer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAEL,KAAK,kBAAkB,EACxB,MAAM,2CAA2C,CAAC;AAGnD,OAAO,EAAwB,KAAK,WAAW,EAAC,MAAM,yBAAyB,CAAC;AAMhF,MAAM,MAAM,+BAA+B,GAAG;IAC5C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,0EAA0E;IAC1E,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,eAAe,EAAE,SAAS,WAAW,EAAE,CAAC;CAClD,CAAC;AAEF,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,SAAkB,IAAI,mCAAmC;CAC1D;AAWD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,qBAAqB;;IAYhC,YAAY,EAAE,EAAE,eAAe,EAc9B;IAED,KAAK,CACH,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,YAAY,SAAwC,EACpD,IAAI,UAAO,GACV,IAAI,CAmCN;IAED;;;;;;;;;OASG;IACH,MAAM,CACJ,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,kBAAkB,EAC1B,YAAY,SAA2C,GACtD,IAAI,CAwBN;IAED,MAAM,CACJ,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EACnB,YAAY,SAAyC,GACpD,+BAA+B,CA2CjC;IAED;;;;OAIG;IACH,QAAQ,IAAI,IAAI,CAMf;CAmBF"}

package/out/zero-cache/src/services/replicator/change-log-stream-writer.js

@@ -50,22 +50,24 @@ var ChangeLogStreamWriter = class {
     `);
 		this.#cookies = new ChangeLogCookieWriter(db.db);
 	}
-	begin(watermark, json) {
+	begin(watermark, json, storedChange = extractChangeSubstring(json, "begin"), hash = true) {
 		assertInvariant(this.#watermark === void 0, `change-log stream transaction already open at ${this.#watermark}`);
 		this.#db.beginImmediate();
 		this.#watermark = watermark;
 		this.#pos = 0;
-		const change = extractChangeSubstring(json, "begin");
+		const change = storedChange;
 		const estimatedBytes = estimateChangeLogStreamRowBytes(watermark, "begin", change);
 		this.#insertChange.run(watermark, this.#pos, "begin", estimatedBytes, change);
-		this.#hasher = new ChangeLogTransactionHasher();
-		this.#hasher.add({
-			watermark,
-			pos: this.#pos,
-			tag: "begin",
-			change,
-			precommit: null
-		});
+		if (hash) {
+			this.#hasher = new ChangeLogTransactionHasher();
+			this.#hasher.add({
+				watermark,
+				pos: this.#pos,
+				tag: "begin",
+				change,
+				precommit: null
+			});
+		}
 		this.#estimatedBytes = estimatedBytes;
 	}
 	/**
@@ -78,13 +80,13 @@ var ChangeLogStreamWriter = class {
 	* has to flush its buffered batch to get the same ordering
 	* (`Storer.#processQueue()`); SQLite gets it from statement order.
 	*/
-	append(json, change) {
+	append(json, change, storedChange = extractChangeSubstring(json, change.tag)) {
 		const watermark = this.#requireWatermark();
 		const { tag } = change;
-		const stored = extractChangeSubstring(json, tag);
+		const stored = storedChange;
 		const estimatedBytes = estimateChangeLogStreamRowBytes(watermark, tag, stored);
 		this.#insertChange.run(watermark, ++this.#pos, tag, estimatedBytes, stored);
-		this.#requireHasher().add({
+		this.#hasher?.add({
 			watermark,
 			pos: this.#pos,
 			tag,
@@ -94,14 +96,14 @@ var ChangeLogStreamWriter = class {
 		this.#estimatedBytes += estimatedBytes;
 		if (isSchemaChange(change)) for (const op of this.#cookies.apply(change)) this.#cookieMutations.push(op.op);
 	}
-	commit(watermark, json, writeTimeMs) {
+	commit(watermark, json, writeTimeMs, storedChange = extractChangeSubstring(json, "commit")) {
 		const precommit = this.#requireWatermark();
 		assertInvariant(watermark === precommit, `change-log stream commit ${watermark} does not match begin ${precommit}`);
-		const change = extractChangeSubstring(json, "commit");
+		const change = storedChange;
 		const estimatedBytes = estimateChangeLogStreamRowBytes(watermark, "commit", change, precommit, true);
 		this.#insertCommit.run(watermark, ++this.#pos, estimatedBytes, change, precommit, writeTimeMs);
-		const hasher = this.#requireHasher();
-		hasher.add({
+		const hasher = this.#hasher;
+		hasher?.add({
 			watermark,
 			pos: this.#pos,
 			tag: "commit",
@@ -111,7 +113,7 @@ var ChangeLogStreamWriter = class {
 		const stats = {
 			rows: this.#pos + 1,
 			estimatedBytes: this.#estimatedBytes + estimatedBytes,
-			hash: hasher.digest(),
+			hash: hasher?.digest(),
 			cookieMutations: this.#cookieMutations
 		};
 		this.#db.commit();
@@ -139,10 +141,6 @@ var ChangeLogStreamWriter = class {
 		assertInvariant(this.#watermark !== void 0, "change-log stream message received outside of a transaction");
 		return this.#watermark;
 	}
-	#requireHasher() {
-		assertInvariant(this.#hasher !== void 0, "change-log stream hash received outside of a transaction");
-		return this.#hasher;
-	}
 };
 //#endregion
 export { ChangeLogStreamWriter };

package/out/zero-cache/src/services/replicator/change-log-stream-writer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-stream-writer.js","names":["#db","#insertChange","#insertCommit","#cookies","#watermark","#pos","#hasher","#estimatedBytes","#requireWatermark","#requireHasher","#cookieMutations","#reset"],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-stream-writer.ts"],"sourcesContent":["import type {Statement} from '../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport {\n  isSchemaChange,\n  type DataOrSchemaChange,\n} from '../change-source/protocol/current/data.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {ChangeLogCookieWriter, type CookieOpTag} from './change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  estimateChangeLogStreamRowBytes,\n} from './change-log-db.ts';\n\nexport type ChangeLogStreamTransactionStats = {\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly hash: string;\n  /**\n   * The cookie mutations this transaction's schema changes folded in, in stream\n   * order. Empty for the overwhelming majority of transactions.\n   */\n  readonly cookieMutations: readonly CookieOpTag[];\n};\n\nexport class ChangeLogStreamInvariantError extends Error {\n  override readonly name = 'ChangeLogStreamInvariantError';\n}\n\nfunction assertInvariant(\n  condition: unknown,\n  message: string,\n): asserts condition {\n  if (!condition) {\n    throw new ChangeLogStreamInvariantError(message);\n  }\n}\n\n/**\n * Appends the canonical downstream stream to the change-log database, which is\n * a separate file from the replica, and folds its schema changes into the log's\n * cookie jar (`change-log-cookies.ts`) as it goes.\n *\n * This class owns the change log's transaction, and its transaction alone. It is\n * driven from the change-streamer's stream loop, which commits it before\n * forwarding a transaction's `commit` message; see `SQLiteChangeLogWriter` in\n * `change-streamer/sqlite-change-log-writer.ts` for the ordering that makes a\n * hole unreachable, and for the fail-soft policy that wraps every call here.\n *\n * Because the cookies are written in that same transaction, they are atomic with\n * the head and {@link rollback} discards them with the rows they were folded\n * from. Neither needs any handling of its own.\n */\nexport class ChangeLogStreamWriter {\n  readonly #db: StatementRunner;\n  readonly #insertChange: Statement;\n  readonly #insertCommit: Statement;\n  readonly #cookies: ChangeLogCookieWriter;\n\n  #watermark: string | undefined;\n  #pos = 0;\n  #estimatedBytes = 0;\n  #hasher: ChangeLogTransactionHasher | undefined;\n  #cookieMutations: CookieOpTag[] = [];\n\n  constructor(db: StatementRunner) {\n    this.#db = db;\n    this.#insertChange = db.db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n        (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\")\n        VALUES (?, ?, ?, ?, ?)\n    `);\n    this.#insertCommit = db.db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n        (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\", \"precommit\",\n         \"writeTimeMs\")\n        VALUES (?, ?, 'commit', ?, ?, ?, ?)\n    `);\n    this.#cookies = new ChangeLogCookieWriter(db.db);\n  }\n\n  begin(watermark: string, json: string): void {\n    assertInvariant(\n      this.#watermark === undefined,\n      `change-log stream transaction already open at ${this.#watermark}`,\n    );\n    // This process is the sole writer of the change-log database, so the write\n    // lock is taken up front rather than risking a failed upgrade from a\n    // deferred transaction partway through the append.\n    this.#db.beginImmediate();\n    this.#watermark = watermark;\n    this.#pos = 0;\n    const change = extractChangeSubstring(json, 'begin');\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      'begin',\n      change,\n    );\n    this.#insertChange.run(\n      watermark,\n      this.#pos,\n      'begin',\n      estimatedBytes,\n      change,\n    );\n    this.#hasher = new ChangeLogTransactionHasher();\n    this.#hasher.add({\n      watermark,\n      pos: this.#pos,\n      tag: 'begin',\n      change,\n      precommit: null,\n    });\n    this.#estimatedBytes = estimatedBytes;\n  }\n\n  /**\n   * Appends one data or schema change. The parsed message is taken rather than\n   * just its tag because a schema change also mutates the cookie jar, and the\n   * caller already holds it — so the fold costs no re-parse.\n   *\n   * The cookie statements run here, i.e. inside this transaction and in stream\n   * order relative to the rows they were folded from. The Postgres change log\n   * has to flush its buffered batch to get the same ordering\n   * (`Storer.#processQueue()`); SQLite gets it from statement order.\n   */\n  append(json: string, change: DataOrSchemaChange): void {\n    const watermark = this.#requireWatermark();\n    const {tag} = change;\n    const stored = extractChangeSubstring(json, tag);\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      tag,\n      stored,\n    );\n    this.#insertChange.run(watermark, ++this.#pos, tag, estimatedBytes, stored);\n    this.#requireHasher().add({\n      watermark,\n      pos: this.#pos,\n      tag,\n      change: stored,\n      precommit: null,\n    });\n    this.#estimatedBytes += estimatedBytes;\n\n    if (isSchemaChange(change)) {\n      for (const op of this.#cookies.apply(change)) {\n        this.#cookieMutations.push(op.op);\n      }\n    }\n  }\n\n  commit(\n    watermark: string,\n    json: string,\n    writeTimeMs: number,\n  ): ChangeLogStreamTransactionStats {\n    const precommit = this.#requireWatermark();\n    assertInvariant(\n      watermark === precommit,\n      `change-log stream commit ${watermark} does not match begin ${precommit}`,\n    );\n    const change = extractChangeSubstring(json, 'commit');\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      'commit',\n      change,\n      precommit,\n      true,\n    );\n    this.#insertCommit.run(\n      watermark,\n      ++this.#pos,\n      estimatedBytes,\n      change,\n      precommit,\n      writeTimeMs,\n    );\n    const hasher = this.#requireHasher();\n    hasher.add({\n      watermark,\n      pos: this.#pos,\n      tag: 'commit',\n      change,\n      precommit,\n    });\n    const stats = {\n      rows: this.#pos + 1,\n      estimatedBytes: this.#estimatedBytes + estimatedBytes,\n      hash: hasher.digest(),\n      cookieMutations: this.#cookieMutations,\n    };\n    // The log is durable at `watermark` from here on, and nothing that can\n    // advance the watermark the stream would resume from has run yet. A crash in\n    // that window leaves a transaction the resumed stream re-delivers, which\n    // reconciliation truncates.\n    this.#db.commit();\n    this.#reset();\n    return stats;\n  }\n\n  /**\n   * Discards the open transaction, if any. Safe to call when none is open — the\n   * caller rolls back on an interrupted change stream, which can arrive between\n   * transactions as well as inside one.\n   */\n  rollback(): void {\n    const {inTransaction} = this.#db.db;\n    this.#reset();\n    if (inTransaction) {\n      this.#db.rollback();\n    }\n  }\n\n  #reset(): void {\n    this.#watermark = undefined;\n    this.#pos = 0;\n    this.#estimatedBytes = 0;\n    this.#hasher = undefined;\n    // A fresh array rather than a truncation: the committed stats hold a\n    // reference to the previous one.\n    this.#cookieMutations = [];\n  }\n\n  #requireWatermark(): string {\n    assertInvariant(\n      this.#watermark !== undefined,\n      'change-log stream message received outside of a transaction',\n    );\n    return this.#watermark;\n  }\n\n  #requireHasher(): ChangeLogTransactionHasher {\n    assertInvariant(\n      this.#hasher !== undefined,\n      'change-log stream hash received outside of a transaction',\n    );\n    return this.#hasher;\n  }\n}\n"],"mappings":";;;;;;AAyBA,IAAa,gCAAb,cAAmD,MAAM;CACvD,OAAyB;AAC3B;AAEA,SAAS,gBACP,WACA,SACmB;CACnB,IAAI,CAAC,WACH,MAAM,IAAI,8BAA8B,OAAO;AAEnD;;;;;;;;;;;;;;;;AAiBA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CAEA;CACA,OAAO;CACP,kBAAkB;CAClB;CACA,mBAAkC,CAAC;CAEnC,YAAY,IAAqB;EAC/B,KAAKA,MAAM;EACX,KAAKC,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;KAGxC;EACD,KAAKC,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;;KAIxC;EACD,KAAKC,WAAW,IAAI,sBAAsB,GAAG,EAAE;CACjD;CAEA,MAAM,WAAmB,MAAoB;EAC3C,gBACE,KAAKC,eAAe,KAAA,GACpB,iDAAiD,KAAKA,YACxD;EAIA,KAAKJ,IAAI,eAAe;EACxB,KAAKI,aAAa;EAClB,KAAKC,OAAO;EACZ,MAAM,SAAS,uBAAuB,MAAM,OAAO;EACnD,MAAM,iBAAiB,gCACrB,WACA,SACA,MACF;EACA,KAAKJ,cAAc,IACjB,WACA,KAAKI,MACL,SACA,gBACA,MACF;EACA,KAAKC,UAAU,IAAI,2BAA2B;EAC9C,KAAKA,QAAQ,IAAI;GACf;GACA,KAAK,KAAKD;GACV,KAAK;GACL;GACA,WAAW;EACb,CAAC;EACD,KAAKE,kBAAkB;CACzB;;;;;;;;;;;CAYA,OAAO,MAAc,QAAkC;EACrD,MAAM,YAAY,KAAKC,kBAAkB;EACzC,MAAM,EAAC,QAAO;EACd,MAAM,SAAS,uBAAuB,MAAM,GAAG;EAC/C,MAAM,iBAAiB,gCACrB,WACA,KACA,MACF;EACA,KAAKP,cAAc,IAAI,WAAW,EAAE,KAAKI,MAAM,KAAK,gBAAgB,MAAM;EAC1E,KAAKI,eAAe,EAAE,IAAI;GACxB;GACA,KAAK,KAAKJ;GACV;GACA,QAAQ;GACR,WAAW;EACb,CAAC;EACD,KAAKE,mBAAmB;EAExB,IAAI,eAAe,MAAM,GACvB,KAAK,MAAM,MAAM,KAAKJ,SAAS,MAAM,MAAM,GACzC,KAAKO,iBAAiB,KAAK,GAAG,EAAE;CAGtC;CAEA,OACE,WACA,MACA,aACiC;EACjC,MAAM,YAAY,KAAKF,kBAAkB;EACzC,gBACE,cAAc,WACd,4BAA4B,UAAU,wBAAwB,WAChE;EACA,MAAM,SAAS,uBAAuB,MAAM,QAAQ;EACpD,MAAM,iBAAiB,gCACrB,WACA,UACA,QACA,WACA,IACF;EACA,KAAKN,cAAc,IACjB,WACA,EAAE,KAAKG,MACP,gBACA,QACA,WACA,WACF;EACA,MAAM,SAAS,KAAKI,eAAe;EACnC,OAAO,IAAI;GACT;GACA,KAAK,KAAKJ;GACV,KAAK;GACL;GACA;EACF,CAAC;EACD,MAAM,QAAQ;GACZ,MAAM,KAAKA,OAAO;GAClB,gBAAgB,KAAKE,kBAAkB;GACvC,MAAM,OAAO,OAAO;GACpB,iBAAiB,KAAKG;EACxB;EAKA,KAAKV,IAAI,OAAO;EAChB,KAAKW,OAAO;EACZ,OAAO;CACT;;;;;;CAOA,WAAiB;EACf,MAAM,EAAC,kBAAiB,KAAKX,IAAI;EACjC,KAAKW,OAAO;EACZ,IAAI,eACF,KAAKX,IAAI,SAAS;CAEtB;CAEA,SAAe;EACb,KAAKI,aAAa,KAAA;EAClB,KAAKC,OAAO;EACZ,KAAKE,kBAAkB;EACvB,KAAKD,UAAU,KAAA;EAGf,KAAKI,mBAAmB,CAAC;CAC3B;CAEA,oBAA4B;EAC1B,gBACE,KAAKN,eAAe,KAAA,GACpB,6DACF;EACA,OAAO,KAAKA;CACd;CAEA,iBAA6C;EAC3C,gBACE,KAAKE,YAAY,KAAA,GACjB,0DACF;EACA,OAAO,KAAKA;CACd;AACF"}
+{"version":3,"file":"change-log-stream-writer.js","names":["#db","#insertChange","#insertCommit","#cookies","#watermark","#pos","#hasher","#estimatedBytes","#requireWatermark","#cookieMutations","#reset"],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-stream-writer.ts"],"sourcesContent":["import type {Statement} from '../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport {\n  isSchemaChange,\n  type DataOrSchemaChange,\n} from '../change-source/protocol/current/data.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {ChangeLogCookieWriter, type CookieOpTag} from './change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  estimateChangeLogStreamRowBytes,\n} from './change-log-db.ts';\n\nexport type ChangeLogStreamTransactionStats = {\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  /** Present only for transactions selected for shadow-write validation. */\n  readonly hash?: string | undefined;\n  /**\n   * The cookie mutations this transaction's schema changes folded in, in stream\n   * order. Empty for the overwhelming majority of transactions.\n   */\n  readonly cookieMutations: readonly CookieOpTag[];\n};\n\nexport class ChangeLogStreamInvariantError extends Error {\n  override readonly name = 'ChangeLogStreamInvariantError';\n}\n\nfunction assertInvariant(\n  condition: unknown,\n  message: string,\n): asserts condition {\n  if (!condition) {\n    throw new ChangeLogStreamInvariantError(message);\n  }\n}\n\n/**\n * Appends the canonical downstream stream to the change-log database, which is\n * a separate file from the replica, and folds its schema changes into the log's\n * cookie jar (`change-log-cookies.ts`) as it goes.\n *\n * This class owns the change log's transaction, and its transaction alone. It is\n * driven from the change-streamer's stream loop, which commits it before\n * forwarding a transaction's `commit` message; see `SQLiteChangeLogWriter` in\n * `change-streamer/sqlite-change-log-writer.ts` for the ordering that makes a\n * hole unreachable, and for the fail-soft policy that wraps every call here.\n *\n * Because the cookies are written in that same transaction, they are atomic with\n * the head and {@link rollback} discards them with the rows they were folded\n * from. Neither needs any handling of its own.\n */\nexport class ChangeLogStreamWriter {\n  readonly #db: StatementRunner;\n  readonly #insertChange: Statement;\n  readonly #insertCommit: Statement;\n  readonly #cookies: ChangeLogCookieWriter;\n\n  #watermark: string | undefined;\n  #pos = 0;\n  #estimatedBytes = 0;\n  #hasher: ChangeLogTransactionHasher | undefined;\n  #cookieMutations: CookieOpTag[] = [];\n\n  constructor(db: StatementRunner) {\n    this.#db = db;\n    this.#insertChange = db.db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n        (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\")\n        VALUES (?, ?, ?, ?, ?)\n    `);\n    this.#insertCommit = db.db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n        (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\", \"precommit\",\n         \"writeTimeMs\")\n        VALUES (?, ?, 'commit', ?, ?, ?, ?)\n    `);\n    this.#cookies = new ChangeLogCookieWriter(db.db);\n  }\n\n  begin(\n    watermark: string,\n    json: string,\n    storedChange = extractChangeSubstring(json, 'begin'),\n    hash = true,\n  ): void {\n    assertInvariant(\n      this.#watermark === undefined,\n      `change-log stream transaction already open at ${this.#watermark}`,\n    );\n    // This process is the sole writer of the change-log database, so the write\n    // lock is taken up front rather than risking a failed upgrade from a\n    // deferred transaction partway through the append.\n    this.#db.beginImmediate();\n    this.#watermark = watermark;\n    this.#pos = 0;\n    const change = storedChange;\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      'begin',\n      change,\n    );\n    this.#insertChange.run(\n      watermark,\n      this.#pos,\n      'begin',\n      estimatedBytes,\n      change,\n    );\n    if (hash) {\n      this.#hasher = new ChangeLogTransactionHasher();\n      this.#hasher.add({\n        watermark,\n        pos: this.#pos,\n        tag: 'begin',\n        change,\n        precommit: null,\n      });\n    }\n    this.#estimatedBytes = estimatedBytes;\n  }\n\n  /**\n   * Appends one data or schema change. The parsed message is taken rather than\n   * just its tag because a schema change also mutates the cookie jar, and the\n   * caller already holds it — so the fold costs no re-parse.\n   *\n   * The cookie statements run here, i.e. inside this transaction and in stream\n   * order relative to the rows they were folded from. The Postgres change log\n   * has to flush its buffered batch to get the same ordering\n   * (`Storer.#processQueue()`); SQLite gets it from statement order.\n   */\n  append(\n    json: string,\n    change: DataOrSchemaChange,\n    storedChange = extractChangeSubstring(json, change.tag),\n  ): void {\n    const watermark = this.#requireWatermark();\n    const {tag} = change;\n    const stored = storedChange;\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      tag,\n      stored,\n    );\n    this.#insertChange.run(watermark, ++this.#pos, tag, estimatedBytes, stored);\n    this.#hasher?.add({\n      watermark,\n      pos: this.#pos,\n      tag,\n      change: stored,\n      precommit: null,\n    });\n    this.#estimatedBytes += estimatedBytes;\n\n    if (isSchemaChange(change)) {\n      for (const op of this.#cookies.apply(change)) {\n        this.#cookieMutations.push(op.op);\n      }\n    }\n  }\n\n  commit(\n    watermark: string,\n    json: string,\n    writeTimeMs: number,\n    storedChange = extractChangeSubstring(json, 'commit'),\n  ): ChangeLogStreamTransactionStats {\n    const precommit = this.#requireWatermark();\n    assertInvariant(\n      watermark === precommit,\n      `change-log stream commit ${watermark} does not match begin ${precommit}`,\n    );\n    const change = storedChange;\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      'commit',\n      change,\n      precommit,\n      true,\n    );\n    this.#insertCommit.run(\n      watermark,\n      ++this.#pos,\n      estimatedBytes,\n      change,\n      precommit,\n      writeTimeMs,\n    );\n    const hasher = this.#hasher;\n    hasher?.add({\n      watermark,\n      pos: this.#pos,\n      tag: 'commit',\n      change,\n      precommit,\n    });\n    const stats = {\n      rows: this.#pos + 1,\n      estimatedBytes: this.#estimatedBytes + estimatedBytes,\n      hash: hasher?.digest(),\n      cookieMutations: this.#cookieMutations,\n    };\n    // The log is durable at `watermark` from here on, and nothing that can\n    // advance the watermark the stream would resume from has run yet. A crash in\n    // that window leaves a transaction the resumed stream re-delivers, which\n    // reconciliation truncates.\n    this.#db.commit();\n    this.#reset();\n    return stats;\n  }\n\n  /**\n   * Discards the open transaction, if any. Safe to call when none is open — the\n   * caller rolls back on an interrupted change stream, which can arrive between\n   * transactions as well as inside one.\n   */\n  rollback(): void {\n    const {inTransaction} = this.#db.db;\n    this.#reset();\n    if (inTransaction) {\n      this.#db.rollback();\n    }\n  }\n\n  #reset(): void {\n    this.#watermark = undefined;\n    this.#pos = 0;\n    this.#estimatedBytes = 0;\n    this.#hasher = undefined;\n    // A fresh array rather than a truncation: the committed stats hold a\n    // reference to the previous one.\n    this.#cookieMutations = [];\n  }\n\n  #requireWatermark(): string {\n    assertInvariant(\n      this.#watermark !== undefined,\n      'change-log stream message received outside of a transaction',\n    );\n    return this.#watermark;\n  }\n}\n"],"mappings":";;;;;;AA0BA,IAAa,gCAAb,cAAmD,MAAM;CACvD,OAAyB;AAC3B;AAEA,SAAS,gBACP,WACA,SACmB;CACnB,IAAI,CAAC,WACH,MAAM,IAAI,8BAA8B,OAAO;AAEnD;;;;;;;;;;;;;;;;AAiBA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CAEA;CACA,OAAO;CACP,kBAAkB;CAClB;CACA,mBAAkC,CAAC;CAEnC,YAAY,IAAqB;EAC/B,KAAKA,MAAM;EACX,KAAKC,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;KAGxC;EACD,KAAKC,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;;KAIxC;EACD,KAAKC,WAAW,IAAI,sBAAsB,GAAG,EAAE;CACjD;CAEA,MACE,WACA,MACA,eAAe,uBAAuB,MAAM,OAAO,GACnD,OAAO,MACD;EACN,gBACE,KAAKC,eAAe,KAAA,GACpB,iDAAiD,KAAKA,YACxD;EAIA,KAAKJ,IAAI,eAAe;EACxB,KAAKI,aAAa;EAClB,KAAKC,OAAO;EACZ,MAAM,SAAS;EACf,MAAM,iBAAiB,gCACrB,WACA,SACA,MACF;EACA,KAAKJ,cAAc,IACjB,WACA,KAAKI,MACL,SACA,gBACA,MACF;EACA,IAAI,MAAM;GACR,KAAKC,UAAU,IAAI,2BAA2B;GAC9C,KAAKA,QAAQ,IAAI;IACf;IACA,KAAK,KAAKD;IACV,KAAK;IACL;IACA,WAAW;GACb,CAAC;EACH;EACA,KAAKE,kBAAkB;CACzB;;;;;;;;;;;CAYA,OACE,MACA,QACA,eAAe,uBAAuB,MAAM,OAAO,GAAG,GAChD;EACN,MAAM,YAAY,KAAKC,kBAAkB;EACzC,MAAM,EAAC,QAAO;EACd,MAAM,SAAS;EACf,MAAM,iBAAiB,gCACrB,WACA,KACA,MACF;EACA,KAAKP,cAAc,IAAI,WAAW,EAAE,KAAKI,MAAM,KAAK,gBAAgB,MAAM;EAC1E,KAAKC,SAAS,IAAI;GAChB;GACA,KAAK,KAAKD;GACV;GACA,QAAQ;GACR,WAAW;EACb,CAAC;EACD,KAAKE,mBAAmB;EAExB,IAAI,eAAe,MAAM,GACvB,KAAK,MAAM,MAAM,KAAKJ,SAAS,MAAM,MAAM,GACzC,KAAKM,iBAAiB,KAAK,GAAG,EAAE;CAGtC;CAEA,OACE,WACA,MACA,aACA,eAAe,uBAAuB,MAAM,QAAQ,GACnB;EACjC,MAAM,YAAY,KAAKD,kBAAkB;EACzC,gBACE,cAAc,WACd,4BAA4B,UAAU,wBAAwB,WAChE;EACA,MAAM,SAAS;EACf,MAAM,iBAAiB,gCACrB,WACA,UACA,QACA,WACA,IACF;EACA,KAAKN,cAAc,IACjB,WACA,EAAE,KAAKG,MACP,gBACA,QACA,WACA,WACF;EACA,MAAM,SAAS,KAAKC;EACpB,QAAQ,IAAI;GACV;GACA,KAAK,KAAKD;GACV,KAAK;GACL;GACA;EACF,CAAC;EACD,MAAM,QAAQ;GACZ,MAAM,KAAKA,OAAO;GAClB,gBAAgB,KAAKE,kBAAkB;GACvC,MAAM,QAAQ,OAAO;GACrB,iBAAiB,KAAKE;EACxB;EAKA,KAAKT,IAAI,OAAO;EAChB,KAAKU,OAAO;EACZ,OAAO;CACT;;;;;;CAOA,WAAiB;EACf,MAAM,EAAC,kBAAiB,KAAKV,IAAI;EACjC,KAAKU,OAAO;EACZ,IAAI,eACF,KAAKV,IAAI,SAAS;CAEtB;CAEA,SAAe;EACb,KAAKI,aAAa,KAAA;EAClB,KAAKC,OAAO;EACZ,KAAKE,kBAAkB;EACvB,KAAKD,UAAU,KAAA;EAGf,KAAKG,mBAAmB,CAAC;CAC3B;CAEA,oBAA4B;EAC1B,gBACE,KAAKL,eAAe,KAAA,GACpB,6DACF;EACA,OAAO,KAAKA;CACd;AACF"}

package/out/zero-cache/src/services/replicator/change-processor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-processor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAmB1E,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AAiC5D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAUpD,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG,cAAc,CAAC;AASlE,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,cAAc,GAAG,SAAS,CAAC;IAC9C,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3C,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,eAAe;;IAiB1B,YACE,EAAE,EAAE,eAAe,EACnB,IAAI,EAAE,mBAAmB,EACzB,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,KAAK,IAAI,EAQpD;IA2BD,KAAK,CAAC,EAAE,EAAE,UAAU,QAEnB;IAED,8CAA8C;IAC9C,cAAc,CACZ,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,gBAAgB,GAC3B,YAAY,GAAG,IAAI,CAkBrB;CAyIF"}
+{"version":3,"file":"change-processor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAmB1E,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AAsC5D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAUpD,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG,cAAc,CAAC;AASlE,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,cAAc,GAAG,SAAS,CAAC;IAC9C,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3C,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,eAAe;;IAiB1B,YACE,EAAE,EAAE,eAAe,EACnB,IAAI,EAAE,mBAAmB,EACzB,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,KAAK,IAAI,EAQpD;IA2BD,KAAK,CAAC,EAAE,EAAE,UAAU,QAEnB;IAED,8CAA8C;IAC9C,cAAc,CACZ,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,gBAAgB,GAC3B,YAAY,GAAG,IAAI,CAkBrB;CAyIF"}

package/out/zero-cache/src/services/replicator/change-processor.js

@@ -1,4 +1,5 @@
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
+import { deepEqual } from "../../../../shared/src/json.js";
 import { AbortError } from "../../../../shared/src/abort-error.js";
 import { must } from "../../../../shared/src/must.js";
 import { mapEntries } from "../../../../shared/src/objects.js";
@@ -15,6 +16,7 @@ import { ChangeLog } from "./schema/change-log.js";
 import { updateReplicationWatermark } from "./schema/replication-state.js";
 import { id } from "../../types/sql.js";
 import { createLiteIndexStatement, createLiteTableStatement, liteColumnDef } from "../../db/create.js";
+import { acceptBackfill } from "../change-source/protocol/backfill-progress.js";
 import { SqliteError } from "@rocicorp/zero-sqlite3";
 import { constants } from "node:buffer";
 //#region ../zero-cache/src/services/replicator/change-processor.ts
@@ -424,7 +426,9 @@ var TransactionProcessor = class {
 	}
 	processDropIndex(drop) {
 		const name = liteTableName(drop.id);
+		const indexExisted = this.#db.db.prepare(`SELECT tbl_name as tableName FROM sqlite_master WHERE type = 'index' AND name = ?`).get(name);
 		this.#db.db.exec(`DROP INDEX IF EXISTS ${id(name)}`);
+		if (indexExisted) this.#logResetOp(indexExisted.tableName);
 		this.#lc.info?.(drop.tag, name);
 	}
 	#bumpVersions(table) {
@@ -460,11 +464,18 @@ var TransactionProcessor = class {
 		}
 		this.#reloadTableSpecs();
 	}
-	processBackfill({ relation, watermark, columns, rowValues }) {
+	processBackfill({ relation, watermark, columns, rowValues, progressMarks }) {
 		const tableName = liteTableName(relation);
 		const tableSpec = must(this.#tableSpecs.get(tableName));
 		const rowKeyCols = relation.rowKey.columns;
 		const cols = [...rowKeyCols, ...columns];
+		const { accepted, ignored } = this.#trackBackfillProgress(relation, cols, (own) => acceptBackfill(own, progressMarks));
+		if (accepted.length === 0) {
+			this.#lc.debug?.(ignored.length ? `skipping backfill of ${rowValues.length} rows into ${tableName}: not contiguous with the progress of columns [${ignored.join(",")}]` : `skipping redundant backfill of ${rowValues.length} rows into ${tableName} (all columns already published)`, progressMarks);
+			return;
+		}
+		if (ignored.length) this.#lc.debug?.(`backfill of ${tableName} is not contiguous with the progress of columns [${ignored.join(",")}]`, progressMarks);
+		const acceptedSet = new Set(accepted);
 		const insertColsStr = [...cols, ZERO_VERSION_COLUMN_NAME].map(id).join(",");
 		const qMarks = Array.from({ length: cols.length + 1 }).fill("?").join(",");
 		const rowKeyColsStr = rowKeyCols.map(id).join(",");
@@ -478,7 +489,7 @@ var TransactionProcessor = class {
 				skipped++;
 				continue;
 			}
-			const updates = rowOp?.op === "s" ? cols.filter((c) => (rowOp.backfillingColumnVersions[c] ?? "") <= watermark) : cols;
+			const updates = (rowOp?.op === "s" ? cols.filter((c) => (rowOp.backfillingColumnVersions[c] ?? "") <= watermark) : cols).filter((c) => acceptedSet.has(c));
 			if (updates.length === 0) {
 				skipped++;
 				continue;
@@ -494,13 +505,47 @@ var TransactionProcessor = class {
 		this.#lc.debug?.(`backfilled ${backfilled} rows (skipped ${skipped}) into ${tableName}`);
 	}
 	#completedBackfill;
+	/**
+	* Applies `accept` to the tracked progress of each of the `cols` that are
+	* still backfilling on this replica, updating the progress of the accepted
+	* ones. Columns that are not backfilling (e.g. already published) are
+	* neither accepted nor ignored.
+	*/
+	#trackBackfillProgress(relation, cols, accept) {
+		const tracked = this.#backfilling.getProgress(relation);
+		const accepted = [];
+		const ignored = [];
+		for (const col of cols) {
+			if (!tracked.has(col)) continue;
+			const progress = tracked.get(col);
+			const result = accept(progress);
+			if (!result.accept) {
+				ignored.push(col);
+				continue;
+			}
+			accepted.push(col);
+			if (!deepEqual(result.progress, progress)) this.#backfilling.setProgress(relation, col, result.progress);
+		}
+		return {
+			accepted,
+			ignored
+		};
+	}
 	processBackfillCompleted(msg) {
-		const { relation, columns, status } = msg;
+		const { relation, columns, status, progressMarks } = msg;
 		const tableName = liteTableName(relation);
-		const cols = [...relation.rowKey.columns, ...columns];
+		const rowKeyCols = relation.rowKey.columns;
+		const { accepted, ignored } = this.#trackBackfillProgress(relation, [...rowKeyCols, ...columns], (own) => acceptBackfill(own, progressMarks ? { previous: progressMarks.previous } : void 0));
+		if (ignored.length) this.#lc.info?.(`ignoring backfill-completed of ${tableName} for columns [${ignored.join(",")}], which have not received all of the data`, progressMarks);
+		if (accepted.length === 0) {
+			if (ignored.length === 0) this.#lc.debug?.(`skipping redundant backfill-completed for ${tableName} (all columns already published)`);
+			return;
+		}
+		const ignoredSet = new Set(ignored);
+		const cols = [...rowKeyCols.filter((c) => !ignoredSet.has(c)), ...columns.filter((c) => !ignoredSet.has(c))];
 		const columnMetadata = must(ColumnMetadataStore.getInstance(this.#db.db));
 		for (const col of cols) columnMetadata.clearBackfilling(tableName, col);
-		this.#backfilling.apply(msg);
+		this.#backfilling.complete(relation, cols);
 		this.#bumpVersions(relation);
 		if (status) this.#completedBackfill = {
 			table: tableName,
@@ -531,6 +576,7 @@ var TransactionProcessor = class {
 	abort(lc) {
 		lc.info?.(`aborting transaction ${this.#version}`);
 		this.#db.rollback();
+		this.#tableSpecs.clear();
 	}
 };
 function differentStorageTypes(oldSpec, newSpec) {