@rocicorp/zero 1.11.0-canary.3 → 1.11.0-canary.30

package/out/zero-cache/src/services/replicator/schema/backfilling.js

@@ -11,6 +11,7 @@ var CREATE_BACKFILLING_TABLE = `
     "table"    TEXT NOT NULL,
     "column"   TEXT NOT NULL,
     "backfill" TEXT NOT NULL,
+    "progress" TEXT,
     PRIMARY KEY ("schema", "table", "column")
   );
 `;
@@ -40,6 +41,8 @@ var BackfillingTracker = class {
 	#dropTable;
 	#renameColumn;
 	#dropColumn;
+	#getProgress;
+	#setProgress;
 	constructor(db) {
 		this.#db = db;
 	}
@@ -62,7 +65,7 @@ var BackfillingTracker = class {
           INSERT INTO "${BACKFILLING_TABLE}"
             ("schema", "table", "column", "backfill") VALUES (?, ?, ?, ?)
             ON CONFLICT ("schema", "table", "column")
-            DO UPDATE SET "backfill" = excluded."backfill"
+            DO UPDATE SET "backfill" = excluded."backfill", "progress" = NULL
         `)).run(op.table.schema, op.table.name, op.column, BigIntJSON.stringify(op.backfill));
 				break;
 			case "rename-table":
@@ -92,6 +95,36 @@ var BackfillingTracker = class {
 			default: unreachable(op);
 		}
 	}
+	/**
+	* Returns the tracked progress of the backfilling columns of the `table`,
+	* keyed by column name. Columns that are not backfilling are absent.
+	*/
+	getProgress(table) {
+		const rows = (this.#getProgress ??= this.#db.prepare(`
+      SELECT "column", "progress" FROM "${BACKFILLING_TABLE}"
+        WHERE "schema" = ? AND "table" = ?
+    `)).all(table.schema, table.name);
+		return new Map(rows.map(({ column, progress }) => [column, progress === null ? void 0 : BigIntJSON.parse(progress)]));
+	}
+	/** Sets (or clears) the tracked progress of a backfilling column. */
+	setProgress(table, column, progress) {
+		(this.#setProgress ??= this.#db.prepare(`
+      UPDATE "${BACKFILLING_TABLE}" SET "progress" = ?
+        WHERE "schema" = ? AND "table" = ? AND "column" = ?
+    `)).run(progress === void 0 ? null : BigIntJSON.stringify(progress), table.schema, table.name, column);
+	}
+	/**
+	* Marks the backfill of the `columns` as complete. This is the subset of
+	* a `backfill-completed` message's columns (including row key columns)
+	* whose progress is contiguous with the message.
+	*/
+	complete(table, columns) {
+		this.#run({
+			op: "complete-backfill",
+			table,
+			columns
+		});
+	}
 	#deleteColumn(table, column) {
 		(this.#dropColumn ??= this.#db.prepare(`
       DELETE FROM "${BACKFILLING_TABLE}"
@@ -111,6 +144,25 @@ function readBackfillRequests(db) {
 	return backfillRequestsFrom(readReplicaCookies(db));
 }
 /**
+* Returns the {@link BackfillRequest}s of the replica's pending backfills,
+* including the tracked progress of each column. These are the backfills that
+* a subscriber reports to the change-streamer when starting a subscription.
+*
+* The caller must pair this snapshot with the replica state version.
+*/
+function readReplicaBackfills(db) {
+	const requests = readBackfillRequests(db);
+	const progress = db.prepare(`
+      SELECT "schema", "table", "column", "progress"
+        FROM "${BACKFILLING_TABLE}" WHERE "progress" IS NOT NULL
+    `).all();
+	for (const { schema, table, column, progress: mark } of progress) {
+		const col = requests.find((r) => r.table.schema === schema && r.table.name === table)?.columns[column];
+		if (col) col.progress = BigIntJSON.parse(mark);
+	}
+	return requests;
+}
+/**
 * The replica's whole cookie set, in the shape the change log holds it — the
 * seed a change log would be initialized from, and the replica-derived half of
 * the initialization comparison.
@@ -209,6 +261,6 @@ function populateBackfillingFromColumnMetadata(lc, db) {
 	lc.info?.(`seeded "${BACKFILLING_TABLE}" with ${backfilling.length} in-flight backfill(s), ${unresolved} of which could not be resolved`);
 }
 //#endregion
-export { BackfillingTracker, CREATE_BACKFILLING_TABLE, populateBackfillingFromColumnMetadata, readBackfillRequests, readReplicaCookies };
+export { BACKFILLING_TABLE, BackfillingTracker, CREATE_BACKFILLING_TABLE, populateBackfillingFromColumnMetadata, readBackfillRequests, readReplicaBackfills, readReplicaCookies };
 
 //# sourceMappingURL=backfilling.js.map

package/out/zero-cache/src/services/replicator/schema/backfilling.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfilling.js","names":["#db","#run","#upsert","#renameTable","#dropTable","#renameColumn","#deleteColumn","#dropColumn"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/backfilling.ts"],"sourcesContent":["/**\n * The replica's backfill cookie, keyed by upstream identity.\n *\n * `_zero.column_metadata.backfill` already records which columns are being\n * backfilled, and it stays authoritative for `isBackfilling`, which is what the\n * replica's own read paths use. What it cannot do is name the table an upstream\n * would recognize: it is keyed by *lite* table name, and `liteTableName()`\n * collapses the `public` schema and has no inverse anywhere in the tree. A lite\n * name containing a dot is ambiguous — `{public, \"a.b\"}` and `{a, b}` produce\n * the same string — and a table with backfilling columns and no\n * `_zero.tableMetadata` row (legal, since its metadata is optional) cannot be\n * resolved at all.\n *\n * That is the whole of what blocks a change log from being initialized out of a\n * restored replica, so this table carries the identity the reconstruction needs:\n * a straight row copy into the change log's own cookie tables, with nothing to\n * reconstruct and nothing to guess. The replica's copy is the *seed* source; the\n * change log's copy (`replicator/change-log-cookies.ts`) is the *working* set,\n * folded forward from the seed as the log's head advances.\n *\n * The cost is a second copy of `backfill` inside the replica, taken\n * deliberately — though not to save writes: `_zero.column_metadata` is written\n * only by initial sync, the DDL handlers, and backfill completion, so two more\n * columns on it would be free. It is a matter of shape. That table is keyed by\n * lite table name, so `schema`/`table` would be non-key duplicates of its own\n * key, to be held in sync with it across every rename with no answer to which\n * encoding is authoritative. This one is keyed exactly as `cdc.backfilling` and\n * `_zero.changeLogBackfilling` are, which is what makes the change log's\n * initialization comparison a row-set diff rather than a translation, and what\n * lets all three stores interpret the same fold. It also holds in-flight backfills and nothing else,\n * where the same query against `column_metadata` is an unindexed scan over a row\n * per column of every table.\n *\n * This table is write-only from the replica's perspective. Nothing in the\n * replicator reads it; it exists to be read by the change-streamer.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../../shared/src/bigint-json.ts';\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {getOrCreateCounter} from '../../../observability/metrics.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport type {\n  BackfillID,\n  Identifier,\n  SchemaChange,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {BackfillRequest} from '../../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  cookieOps,\n  type CookieOp,\n  type CookieSet,\n} from '../change-log-cookies.ts';\n\nexport const BACKFILLING_TABLE = '_zero.backfilling';\n\n// `backfill` holds the JSON that `cdc.backfilling` holds as JSONB. SQLite has\n// no JSONB, and nothing here queries into the document: it is stored to be\n// handed back to the change source verbatim.\nexport const CREATE_BACKFILLING_TABLE = /*sql*/ `\n  CREATE TABLE \"${BACKFILLING_TABLE}\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n`;\n\n/**\n * The replica's interpreter of the cookie fold in\n * `replicator/change-log-cookies.ts`.\n *\n * The fold is shared rather than re-implemented for the same reason the\n * Postgres and SQLite change logs share it: the three stores must agree on every\n * transition forever, and the failure mode when they drift is a backfill that is\n * silently never re-requested, on a column that is then silently never\n * populated. Sharing it makes the replica-versus-Postgres comparison a\n * regression test on the interpreters rather than on three implementations of\n * the same switch.\n *\n * Only the backfill half of the fold is interpreted here. The replica's metadata\n * cookie is `_zero.tableMetadata.upstreamMetadata`, which `TableMetadataTracker`\n * already maintains at the same sites, so an `upsert-metadata` op is inert.\n *\n * Statements are lazily prepared, matching `TableMetadataTracker`: a replica\n * that never sees a schema change never prepares any of them.\n */\nexport class BackfillingTracker {\n  readonly #db: Database;\n\n  #upsert: Statement | undefined;\n  #renameTable: Statement | undefined;\n  #dropTable: Statement | undefined;\n  #renameColumn: Statement | undefined;\n  #dropColumn: Statement | undefined;\n\n  constructor(db: Database) {\n    this.#db = db;\n  }\n\n  /**\n   * Applies the change's backfill-cookie ops, returning the ops that were\n   * applied. Changes that carry no backfill state — index changes, metadata-only\n   * updates, and the `create-table` / `add-column` variants from a change source\n   * that does not support backfill — apply nothing.\n   */\n  apply(change: SchemaChange): CookieOp[] {\n    const ops = cookieOps(change);\n    for (const op of ops) {\n      this.#run(op);\n    }\n    return ops;\n  }\n\n  #run(op: CookieOp): void {\n    switch (op.op) {\n      case 'upsert-metadata':\n        // Maintained by TableMetadataTracker in \"_zero.tableMetadata\".\n        break;\n\n      case 'upsert-backfill':\n        (this.#upsert ??= this.#db.prepare(/*sql*/ `\n          INSERT INTO \"${BACKFILLING_TABLE}\"\n            (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n            ON CONFLICT (\"schema\", \"table\", \"column\")\n            DO UPDATE SET \"backfill\" = excluded.\"backfill\"\n        `)).run(\n          op.table.schema,\n          op.table.name,\n          op.column,\n          BigIntJSON.stringify(op.backfill),\n        );\n        break;\n\n      case 'rename-table':\n        (this.#renameTable ??= this.#db.prepare(/*sql*/ `\n          UPDATE \"${BACKFILLING_TABLE}\"\n            SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n        `)).run(op.new.schema, op.new.name, op.old.schema, op.old.name);\n        break;\n\n      case 'drop-table':\n        (this.#dropTable ??= this.#db.prepare(/*sql*/ `\n          DELETE FROM \"${BACKFILLING_TABLE}\"\n            WHERE \"schema\" = ? AND \"table\" = ?\n        `)).run(op.table.schema, op.table.name);\n        break;\n\n      case 'rename-column':\n        (this.#renameColumn ??= this.#db.prepare(/*sql*/ `\n          UPDATE \"${BACKFILLING_TABLE}\" SET \"column\" = ?\n            WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n        `)).run(op.new, op.table.schema, op.table.name, op.old);\n        break;\n\n      case 'drop-column':\n        this.#deleteColumn(op.table, op.column);\n        break;\n\n      case 'complete-backfill':\n        // A per-column delete rather than an `IN` list, which cannot be a\n        // single prepared statement across arities. The columns of one\n        // completed backfill are few, and this runs only on schema changes.\n        for (const column of op.columns) {\n          this.#deleteColumn(op.table, column);\n        }\n        break;\n\n      default:\n        unreachable(op);\n    }\n  }\n\n  #deleteColumn(table: Identifier, column: string): void {\n    (this.#dropColumn ??= this.#db.prepare(/*sql*/ `\n      DELETE FROM \"${BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `)).run(table.schema, table.name, column);\n  }\n}\n\n/**\n * Returns the {@link BackfillRequest}s from the current replica cookies. The\n * result has the same shape as the Postgres initialization result.\n *\n * The caller must pair this snapshot with the replica state version. When the\n * replica trails the change log, the comparison applies the missing schema\n * changes before it compares the results.\n */\nexport function readBackfillRequests(db: Database): BackfillRequest[] {\n  return backfillRequestsFrom(readReplicaCookies(db));\n}\n\n/**\n * The replica's whole cookie set, in the shape the change log holds it — the\n * seed a change log would be initialized from, and the replica-derived half of\n * the initialization comparison.\n *\n * The metadata half is filtered on `upstreamMetadata IS NOT NULL` because\n * `_zero.tableMetadata` carries `minRowVersion` as well, so a row exists for\n * every table whose rows were ever force-re-downloaded, whether or not upstream\n * ever sent metadata for it. Neither cookie store can represent that row:\n * `cdc.tableMetadata.\"metadata\"` and `_zero.changeLogTableMetadata.\"metadata\"`\n * are both `NOT NULL`.\n *\n * Only meaningful paired with `_zero.replicationState.stateVersion`, read in\n * the same snapshot (invariant 15).\n */\nexport function readReplicaCookies(db: Database): CookieSet {\n  const tableMetadata = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"upstreamMetadata\" AS \"metadata\"\n        FROM \"_zero.tableMetadata\"\n        WHERE \"upstreamMetadata\" IS NOT NULL\n        ORDER BY \"schema\", \"table\"\n    `)\n    .all<{schema: string; table: string; metadata: string}>()\n    .map(({schema, table, metadata}) => ({\n      schema,\n      table,\n      metadata: BigIntJSON.parse(metadata) as TableMetadata,\n    }));\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"column\", \"backfill\"\n        FROM \"${BACKFILLING_TABLE}\"\n        ORDER BY \"schema\", \"table\", \"column\"\n    `)\n    .all<{schema: string; table: string; column: string; backfill: string}>()\n    .map(({schema, table, column, backfill}) => ({\n      schema,\n      table,\n      column,\n      backfill: BigIntJSON.parse(backfill) as BackfillID,\n    }));\n\n  return {tableMetadata, backfilling};\n}\n\n/**\n * Seeds {@link BACKFILLING_TABLE} from `_zero.column_metadata`, resolving each\n * lite table name back to its upstream identity. Used by the v17 migration.\n *\n * The resolution is exact for a table that has a `_zero.tableMetadata` row, and\n * exact for a lite name with no dot in it — `liteTableName()` only omits the\n * schema for `public`. What is left over is an in-flight backfill, on a\n * non-`public` schema, on a table with no metadata row, at the instant of\n * upgrade. Those get the best-effort first-dot split and are counted, because a\n * wrong identity produces a `BackfillRequest` the change source will reject\n * rather than a silently wrong one, and because the initialization comparison\n * surfaces it before anything depends on it. The expected count is zero.\n *\n * A full replace rather than an insert-if-absent: `migrateData` re-runs after a\n * rollback and roll-forward, during which a v16 zero-cache advanced the replica\n * without knowing this table existed. Rebuilding it from `column_metadata` —\n * which that zero-cache *did* maintain — is what makes the re-run both\n * idempotent and correct.\n */\nexport function populateBackfillingFromColumnMetadata(\n  lc: LogContext,\n  db: Database,\n): void {\n  db.prepare(/*sql*/ `DELETE FROM \"${BACKFILLING_TABLE}\"`).run();\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT table_name, column_name, backfill FROM \"_zero.column_metadata\"\n        WHERE backfill IS NOT NULL\n        ORDER BY table_name, column_name\n    `)\n    .all<{table_name: string; column_name: string; backfill: string}>();\n\n  const unresolvable = getOrCreateCounter(\n    'replica',\n    'backfilling_unresolvable_rows',\n    'In-flight backfills whose upstream table identity could not be recovered ' +\n      'from the replica when \"' +\n      BACKFILLING_TABLE +\n      '\" was created. A best-effort identity is stored; the expected count is 0.',\n  );\n  if (backfilling.length === 0) {\n    unresolvable.add(0); // the common case: publish the series anyway\n    return;\n  }\n\n  const byLiteName = new Map<string, Identifier>(\n    db\n      .prepare(/*sql*/ `SELECT \"schema\", \"table\" FROM \"_zero.tableMetadata\"`)\n      .all<{schema: string; table: string}>()\n      .map(({schema, table}) => [\n        liteTableName({schema, name: table}),\n        {schema, name: table},\n      ]),\n  );\n\n  const insert = db.prepare(/*sql*/ `\n    INSERT INTO \"${BACKFILLING_TABLE}\"\n      (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n  `);\n  let unresolved = 0;\n  for (const {table_name: liteName, column_name, backfill} of backfilling) {\n    const known = byLiteName.get(liteName);\n    const dot = known ? -1 : liteName.indexOf('.');\n    const table = known ?? {\n      schema: dot < 0 ? 'public' : liteName.slice(0, dot),\n      name: dot < 0 ? liteName : liteName.slice(dot + 1),\n    };\n    if (!known && dot >= 0) {\n      unresolved++;\n      lc.warn?.(\n        `no metadata for backfilling table \"${liteName}\"; ` +\n          `assuming {schema: \"${table.schema}\", name: \"${table.name}\"}`,\n      );\n    }\n    insert.run(table.schema, table.name, column_name, backfill);\n  }\n  unresolvable.add(unresolved);\n  lc.info?.(\n    `seeded \"${BACKFILLING_TABLE}\" with ${backfilling.length} in-flight ` +\n      `backfill(s), ${unresolved} of which could not be resolved`,\n  );\n}\n"],"mappings":";;;;;;AAyDA,IAAa,oBAAoB;AAKjC,IAAa,2BAAmC;kBAC9B,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4BpC,IAAa,qBAAb,MAAgC;CAC9B;CAEA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAKA,MAAM;CACb;;;;;;;CAQA,MAAM,QAAkC;EACtC,MAAM,MAAM,UAAU,MAAM;EAC5B,KAAK,MAAM,MAAM,KACf,KAAKC,KAAK,EAAE;EAEd,OAAO;CACT;CAEA,KAAK,IAAoB;EACvB,QAAQ,GAAG,IAAX;GACE,KAAK,mBAEH;GAEF,KAAK;IACH,CAAC,KAAKC,YAAY,KAAKF,IAAI,QAAgB;yBAC1B,kBAAkB;;;;SAIlC,GAAG,IACF,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,QACH,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,CAAC,KAAKG,iBAAiB,KAAKH,IAAI,QAAgB;oBACpC,kBAAkB;;SAE7B,GAAG,IAAI,GAAG,IAAI,QAAQ,GAAG,IAAI,MAAM,GAAG,IAAI,QAAQ,GAAG,IAAI,IAAI;IAC9D;GAEF,KAAK;IACH,CAAC,KAAKI,eAAe,KAAKJ,IAAI,QAAgB;yBAC7B,kBAAkB;;SAElC,GAAG,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IACtC;GAEF,KAAK;IACH,CAAC,KAAKK,kBAAkB,KAAKL,IAAI,QAAgB;oBACrC,kBAAkB;;SAE7B,GAAG,IAAI,GAAG,KAAK,GAAG,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,GAAG;IACtD;GAEF,KAAK;IACH,KAAKM,cAAc,GAAG,OAAO,GAAG,MAAM;IACtC;GAEF,KAAK;IAIH,KAAK,MAAM,UAAU,GAAG,SACtB,KAAKA,cAAc,GAAG,OAAO,MAAM;IAErC;GAEF,SACE,YAAY,EAAE;EAClB;CACF;CAEA,cAAc,OAAmB,QAAsB;EACrD,CAAC,KAAKC,gBAAgB,KAAKP,IAAI,QAAgB;qBAC9B,kBAAkB;;KAElC,GAAG,IAAI,MAAM,QAAQ,MAAM,MAAM,MAAM;CAC1C;AACF;;;;;;;;;AAUA,SAAgB,qBAAqB,IAAiC;CACpE,OAAO,qBAAqB,mBAAmB,EAAE,CAAC;AACpD;;;;;;;;;;;;;;;;AAiBA,SAAgB,mBAAmB,IAAyB;CA6B1D,OAAO;EAAC,eA5Bc,GACnB,QAAgB;;;;;KAKhB,EACA,IAAuD,EACvD,KAAK,EAAC,QAAQ,OAAO,gBAAe;GACnC;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAgBM;EAAe,aAdH,GACjB,QAAgB;;gBAEL,kBAAkB;;KAE7B,EACA,IAAuE,EACvE,KAAK,EAAC,QAAQ,OAAO,QAAQ,gBAAe;GAC3C;GACA;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAEqB;CAAW;AACpC;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,sCACd,IACA,IACM;CACN,GAAG,QAAgB,gBAAgB,kBAAkB,EAAE,EAAE,IAAI;CAE7D,MAAM,cAAc,GACjB,QAAgB;;;;KAIhB,EACA,IAAiE;CAEpE,MAAM,eAAe,mBACnB,WACA,iCACA,sGAEE,oBACA,4EACJ;CACA,IAAI,YAAY,WAAW,GAAG;EAC5B,aAAa,IAAI,CAAC;EAClB;CACF;CAEA,MAAM,aAAa,IAAI,IACrB,GACG,QAAgB,qDAAqD,EACrE,IAAqC,EACrC,KAAK,EAAC,QAAQ,YAAW,CACxB,cAAc;EAAC;EAAQ,MAAM;CAAK,CAAC,GACnC;EAAC;EAAQ,MAAM;CAAK,CACtB,CAAC,CACL;CAEA,MAAM,SAAS,GAAG,QAAgB;mBACjB,kBAAkB;;GAElC;CACD,IAAI,aAAa;CACjB,KAAK,MAAM,EAAC,YAAY,UAAU,aAAa,cAAa,aAAa;EACvE,MAAM,QAAQ,WAAW,IAAI,QAAQ;EACrC,MAAM,MAAM,QAAQ,KAAK,SAAS,QAAQ,GAAG;EAC7C,MAAM,QAAQ,SAAS;GACrB,QAAQ,MAAM,IAAI,WAAW,SAAS,MAAM,GAAG,GAAG;GAClD,MAAM,MAAM,IAAI,WAAW,SAAS,MAAM,MAAM,CAAC;EACnD;EACA,IAAI,CAAC,SAAS,OAAO,GAAG;GACtB;GACA,GAAG,OACD,sCAAsC,SAAS,wBACvB,MAAM,OAAO,YAAY,MAAM,KAAK,GAC9D;EACF;EACA,OAAO,IAAI,MAAM,QAAQ,MAAM,MAAM,aAAa,QAAQ;CAC5D;CACA,aAAa,IAAI,UAAU;CAC3B,GAAG,OACD,WAAW,kBAAkB,SAAS,YAAY,OAAO,0BACvC,WAAW,gCAC/B;AACF"}
+{"version":3,"file":"backfilling.js","names":["#db","#run","#upsert","#renameTable","#dropTable","#renameColumn","#deleteColumn","#getProgress","#setProgress","#dropColumn"],"sources":["../../../../../../../zero-cache/src/services/replicator/schema/backfilling.ts"],"sourcesContent":["/**\n * The replica's backfill cookie, keyed by upstream identity.\n *\n * `_zero.column_metadata.backfill` already records which columns are being\n * backfilled, and it stays authoritative for `isBackfilling`, which is what the\n * replica's own read paths use. What it cannot do is name the table an upstream\n * would recognize: it is keyed by *lite* table name, and `liteTableName()`\n * collapses the `public` schema and has no inverse anywhere in the tree. A lite\n * name containing a dot is ambiguous — `{public, \"a.b\"}` and `{a, b}` produce\n * the same string — and a table with backfilling columns and no\n * `_zero.tableMetadata` row (legal, since its metadata is optional) cannot be\n * resolved at all.\n *\n * That is the whole of what blocks a change log from being initialized out of a\n * restored replica, so this table carries the identity the reconstruction needs:\n * a straight row copy into the change log's own cookie tables, with nothing to\n * reconstruct and nothing to guess. The replica's copy is the *seed* source; the\n * change log's copy (`replicator/change-log-cookies.ts`) is the *working* set,\n * folded forward from the seed as the log's head advances.\n *\n * The cost is a second copy of `backfill` inside the replica, taken\n * deliberately — though not to save writes: `_zero.column_metadata` is written\n * only by initial sync, the DDL handlers, and backfill completion, so two more\n * columns on it would be free. It is a matter of shape. That table is keyed by\n * lite table name, so `schema`/`table` would be non-key duplicates of its own\n * key, to be held in sync with it across every rename with no answer to which\n * encoding is authoritative. This one is keyed exactly as `cdc.backfilling` and\n * `_zero.changeLogBackfilling` are, which is what makes the change log's\n * initialization comparison a row-set diff rather than a translation, and what\n * lets all three stores interpret the same fold. It also holds in-flight backfills and nothing else,\n * where the same query against `column_metadata` is an unindexed scan over a row\n * per column of every table.\n *\n * This table is write-only from the replica's perspective. Nothing in the\n * replicator reads it; it exists to be read by the change-streamer.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../../shared/src/bigint-json.ts';\nimport type {Database, Statement} from '../../../../../zqlite/src/db.ts';\nimport {getOrCreateCounter} from '../../../observability/metrics.ts';\nimport {liteTableName} from '../../../types/names.ts';\nimport type {\n  BackfillID,\n  BackfillProgressMark,\n  Identifier,\n  SchemaChange,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {BackfillRequest} from '../../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  cookieOps,\n  type CookieOp,\n  type CookieSet,\n} from '../change-log-cookies.ts';\n\nexport const BACKFILLING_TABLE = '_zero.backfilling';\n\n// `backfill` holds the JSON that `cdc.backfilling` holds as JSONB. SQLite has\n// no JSONB, and nothing here queries into the document: it is stored to be\n// handed back to the change source verbatim.\n//\n// `progress` holds the JSON of the BackfillProgressMark up to which the\n// column's backfill data has been received contiguously (see\n// `acceptBackfill()`), or NULL if no data has been received (i.e. the backfill\n// must start from scratch). It is deliberately a separate column from\n// `backfill` so that the cookie set (and its comparison with the change log's\n// cookie jars) is unaffected by it.\nexport const CREATE_BACKFILLING_TABLE = /*sql*/ `\n  CREATE TABLE \"${BACKFILLING_TABLE}\" (\n    \"schema\"   TEXT NOT NULL,\n    \"table\"    TEXT NOT NULL,\n    \"column\"   TEXT NOT NULL,\n    \"backfill\" TEXT NOT NULL,\n    \"progress\" TEXT,\n    PRIMARY KEY (\"schema\", \"table\", \"column\")\n  );\n`;\n\n/**\n * The replica's interpreter of the cookie fold in\n * `replicator/change-log-cookies.ts`.\n *\n * The fold is shared rather than re-implemented for the same reason the\n * Postgres and SQLite change logs share it: the three stores must agree on every\n * transition forever, and the failure mode when they drift is a backfill that is\n * silently never re-requested, on a column that is then silently never\n * populated. Sharing it makes the replica-versus-Postgres comparison a\n * regression test on the interpreters rather than on three implementations of\n * the same switch.\n *\n * Only the backfill half of the fold is interpreted here. The replica's metadata\n * cookie is `_zero.tableMetadata.upstreamMetadata`, which `TableMetadataTracker`\n * already maintains at the same sites, so an `upsert-metadata` op is inert.\n *\n * Statements are lazily prepared, matching `TableMetadataTracker`: a replica\n * that never sees a schema change never prepares any of them.\n */\nexport class BackfillingTracker {\n  readonly #db: Database;\n\n  #upsert: Statement | undefined;\n  #renameTable: Statement | undefined;\n  #dropTable: Statement | undefined;\n  #renameColumn: Statement | undefined;\n  #dropColumn: Statement | undefined;\n  #getProgress: Statement | undefined;\n  #setProgress: Statement | undefined;\n\n  constructor(db: Database) {\n    this.#db = db;\n  }\n\n  /**\n   * Applies the change's backfill-cookie ops, returning the ops that were\n   * applied. Changes that carry no backfill state — index changes, metadata-only\n   * updates, and the `create-table` / `add-column` variants from a change source\n   * that does not support backfill — apply nothing.\n   */\n  apply(change: SchemaChange): CookieOp[] {\n    const ops = cookieOps(change);\n    for (const op of ops) {\n      this.#run(op);\n    }\n    return ops;\n  }\n\n  #run(op: CookieOp): void {\n    switch (op.op) {\n      case 'upsert-metadata':\n        // Maintained by TableMetadataTracker in \"_zero.tableMetadata\".\n        break;\n\n      case 'upsert-backfill':\n        (this.#upsert ??= this.#db.prepare(/*sql*/ `\n          INSERT INTO \"${BACKFILLING_TABLE}\"\n            (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n            ON CONFLICT (\"schema\", \"table\", \"column\")\n            DO UPDATE SET \"backfill\" = excluded.\"backfill\", \"progress\" = NULL\n        `)).run(\n          op.table.schema,\n          op.table.name,\n          op.column,\n          BigIntJSON.stringify(op.backfill),\n        );\n        break;\n\n      case 'rename-table':\n        (this.#renameTable ??= this.#db.prepare(/*sql*/ `\n          UPDATE \"${BACKFILLING_TABLE}\"\n            SET \"schema\" = ?, \"table\" = ? WHERE \"schema\" = ? AND \"table\" = ?\n        `)).run(op.new.schema, op.new.name, op.old.schema, op.old.name);\n        break;\n\n      case 'drop-table':\n        (this.#dropTable ??= this.#db.prepare(/*sql*/ `\n          DELETE FROM \"${BACKFILLING_TABLE}\"\n            WHERE \"schema\" = ? AND \"table\" = ?\n        `)).run(op.table.schema, op.table.name);\n        break;\n\n      case 'rename-column':\n        (this.#renameColumn ??= this.#db.prepare(/*sql*/ `\n          UPDATE \"${BACKFILLING_TABLE}\" SET \"column\" = ?\n            WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n        `)).run(op.new, op.table.schema, op.table.name, op.old);\n        break;\n\n      case 'drop-column':\n        this.#deleteColumn(op.table, op.column);\n        break;\n\n      case 'complete-backfill':\n        // A per-column delete rather than an `IN` list, which cannot be a\n        // single prepared statement across arities. The columns of one\n        // completed backfill are few, and this runs only on schema changes.\n        for (const column of op.columns) {\n          this.#deleteColumn(op.table, column);\n        }\n        break;\n\n      default:\n        unreachable(op);\n    }\n  }\n\n  /**\n   * Returns the tracked progress of the backfilling columns of the `table`,\n   * keyed by column name. Columns that are not backfilling are absent.\n   */\n  getProgress(\n    table: Identifier,\n  ): Map<string, BackfillProgressMark | undefined> {\n    const rows = (this.#getProgress ??= this.#db.prepare(/*sql*/ `\n      SELECT \"column\", \"progress\" FROM \"${BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ?\n    `)).all<{column: string; progress: string | null}>(\n      table.schema,\n      table.name,\n    );\n    return new Map(\n      rows.map(({column, progress}) => [\n        column,\n        progress === null\n          ? undefined\n          : (BigIntJSON.parse(progress) as BackfillProgressMark),\n      ]),\n    );\n  }\n\n  /** Sets (or clears) the tracked progress of a backfilling column. */\n  setProgress(\n    table: Identifier,\n    column: string,\n    progress: BackfillProgressMark | undefined,\n  ): void {\n    (this.#setProgress ??= this.#db.prepare(/*sql*/ `\n      UPDATE \"${BACKFILLING_TABLE}\" SET \"progress\" = ?\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `)).run(\n      progress === undefined ? null : BigIntJSON.stringify(progress),\n      table.schema,\n      table.name,\n      column,\n    );\n  }\n\n  /**\n   * Marks the backfill of the `columns` as complete. This is the subset of\n   * a `backfill-completed` message's columns (including row key columns)\n   * whose progress is contiguous with the message.\n   */\n  complete(table: Identifier, columns: readonly string[]): void {\n    this.#run({op: 'complete-backfill', table, columns});\n  }\n\n  #deleteColumn(table: Identifier, column: string): void {\n    (this.#dropColumn ??= this.#db.prepare(/*sql*/ `\n      DELETE FROM \"${BACKFILLING_TABLE}\"\n        WHERE \"schema\" = ? AND \"table\" = ? AND \"column\" = ?\n    `)).run(table.schema, table.name, column);\n  }\n}\n\n/**\n * Returns the {@link BackfillRequest}s from the current replica cookies. The\n * result has the same shape as the Postgres initialization result.\n *\n * The caller must pair this snapshot with the replica state version. When the\n * replica trails the change log, the comparison applies the missing schema\n * changes before it compares the results.\n */\nexport function readBackfillRequests(db: Database): BackfillRequest[] {\n  return backfillRequestsFrom(readReplicaCookies(db));\n}\n\n/**\n * Returns the {@link BackfillRequest}s of the replica's pending backfills,\n * including the tracked progress of each column. These are the backfills that\n * a subscriber reports to the change-streamer when starting a subscription.\n *\n * The caller must pair this snapshot with the replica state version.\n */\nexport function readReplicaBackfills(db: Database): BackfillRequest[] {\n  const requests = readBackfillRequests(db);\n  const progress = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"column\", \"progress\"\n        FROM \"${BACKFILLING_TABLE}\" WHERE \"progress\" IS NOT NULL\n    `)\n    .all<{schema: string; table: string; column: string; progress: string}>();\n  for (const {schema, table, column, progress: mark} of progress) {\n    const req = requests.find(\n      r => r.table.schema === schema && r.table.name === table,\n    );\n    const col = req?.columns[column];\n    if (col) {\n      col.progress = BigIntJSON.parse(mark) as BackfillProgressMark;\n    }\n  }\n  return requests;\n}\n\n/**\n * The replica's whole cookie set, in the shape the change log holds it — the\n * seed a change log would be initialized from, and the replica-derived half of\n * the initialization comparison.\n *\n * The metadata half is filtered on `upstreamMetadata IS NOT NULL` because\n * `_zero.tableMetadata` carries `minRowVersion` as well, so a row exists for\n * every table whose rows were ever force-re-downloaded, whether or not upstream\n * ever sent metadata for it. Neither cookie store can represent that row:\n * `cdc.tableMetadata.\"metadata\"` and `_zero.changeLogTableMetadata.\"metadata\"`\n * are both `NOT NULL`.\n *\n * Only meaningful paired with `_zero.replicationState.stateVersion`, read in\n * the same snapshot (invariant 15).\n */\nexport function readReplicaCookies(db: Database): CookieSet {\n  const tableMetadata = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"upstreamMetadata\" AS \"metadata\"\n        FROM \"_zero.tableMetadata\"\n        WHERE \"upstreamMetadata\" IS NOT NULL\n        ORDER BY \"schema\", \"table\"\n    `)\n    .all<{schema: string; table: string; metadata: string}>()\n    .map(({schema, table, metadata}) => ({\n      schema,\n      table,\n      metadata: BigIntJSON.parse(metadata) as TableMetadata,\n    }));\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT \"schema\", \"table\", \"column\", \"backfill\"\n        FROM \"${BACKFILLING_TABLE}\"\n        ORDER BY \"schema\", \"table\", \"column\"\n    `)\n    .all<{schema: string; table: string; column: string; backfill: string}>()\n    .map(({schema, table, column, backfill}) => ({\n      schema,\n      table,\n      column,\n      backfill: BigIntJSON.parse(backfill) as BackfillID,\n    }));\n\n  return {tableMetadata, backfilling};\n}\n\n/**\n * Seeds {@link BACKFILLING_TABLE} from `_zero.column_metadata`, resolving each\n * lite table name back to its upstream identity. Used by the v17 migration.\n *\n * The resolution is exact for a table that has a `_zero.tableMetadata` row, and\n * exact for a lite name with no dot in it — `liteTableName()` only omits the\n * schema for `public`. What is left over is an in-flight backfill, on a\n * non-`public` schema, on a table with no metadata row, at the instant of\n * upgrade. Those get the best-effort first-dot split and are counted, because a\n * wrong identity produces a `BackfillRequest` the change source will reject\n * rather than a silently wrong one, and because the initialization comparison\n * surfaces it before anything depends on it. The expected count is zero.\n *\n * A full replace rather than an insert-if-absent: `migrateData` re-runs after a\n * rollback and roll-forward, during which a v16 zero-cache advanced the replica\n * without knowing this table existed. Rebuilding it from `column_metadata` —\n * which that zero-cache *did* maintain — is what makes the re-run both\n * idempotent and correct.\n */\nexport function populateBackfillingFromColumnMetadata(\n  lc: LogContext,\n  db: Database,\n): void {\n  db.prepare(/*sql*/ `DELETE FROM \"${BACKFILLING_TABLE}\"`).run();\n\n  const backfilling = db\n    .prepare(/*sql*/ `\n      SELECT table_name, column_name, backfill FROM \"_zero.column_metadata\"\n        WHERE backfill IS NOT NULL\n        ORDER BY table_name, column_name\n    `)\n    .all<{table_name: string; column_name: string; backfill: string}>();\n\n  const unresolvable = getOrCreateCounter(\n    'replica',\n    'backfilling_unresolvable_rows',\n    'In-flight backfills whose upstream table identity could not be recovered ' +\n      'from the replica when \"' +\n      BACKFILLING_TABLE +\n      '\" was created. A best-effort identity is stored; the expected count is 0.',\n  );\n  if (backfilling.length === 0) {\n    unresolvable.add(0); // the common case: publish the series anyway\n    return;\n  }\n\n  const byLiteName = new Map<string, Identifier>(\n    db\n      .prepare(/*sql*/ `SELECT \"schema\", \"table\" FROM \"_zero.tableMetadata\"`)\n      .all<{schema: string; table: string}>()\n      .map(({schema, table}) => [\n        liteTableName({schema, name: table}),\n        {schema, name: table},\n      ]),\n  );\n\n  const insert = db.prepare(/*sql*/ `\n    INSERT INTO \"${BACKFILLING_TABLE}\"\n      (\"schema\", \"table\", \"column\", \"backfill\") VALUES (?, ?, ?, ?)\n  `);\n  let unresolved = 0;\n  for (const {table_name: liteName, column_name, backfill} of backfilling) {\n    const known = byLiteName.get(liteName);\n    const dot = known ? -1 : liteName.indexOf('.');\n    const table = known ?? {\n      schema: dot < 0 ? 'public' : liteName.slice(0, dot),\n      name: dot < 0 ? liteName : liteName.slice(dot + 1),\n    };\n    if (!known && dot >= 0) {\n      unresolved++;\n      lc.warn?.(\n        `no metadata for backfilling table \"${liteName}\"; ` +\n          `assuming {schema: \"${table.schema}\", name: \"${table.name}\"}`,\n      );\n    }\n    insert.run(table.schema, table.name, column_name, backfill);\n  }\n  unresolvable.add(unresolved);\n  lc.info?.(\n    `seeded \"${BACKFILLING_TABLE}\" with ${backfilling.length} in-flight ` +\n      `backfill(s), ${unresolved} of which could not be resolved`,\n  );\n}\n"],"mappings":";;;;;;AA0DA,IAAa,oBAAoB;AAYjC,IAAa,2BAAmC;kBAC9B,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BpC,IAAa,qBAAb,MAAgC;CAC9B;CAEA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,IAAc;EACxB,KAAKA,MAAM;CACb;;;;;;;CAQA,MAAM,QAAkC;EACtC,MAAM,MAAM,UAAU,MAAM;EAC5B,KAAK,MAAM,MAAM,KACf,KAAKC,KAAK,EAAE;EAEd,OAAO;CACT;CAEA,KAAK,IAAoB;EACvB,QAAQ,GAAG,IAAX;GACE,KAAK,mBAEH;GAEF,KAAK;IACH,CAAC,KAAKC,YAAY,KAAKF,IAAI,QAAgB;yBAC1B,kBAAkB;;;;SAIlC,GAAG,IACF,GAAG,MAAM,QACT,GAAG,MAAM,MACT,GAAG,QACH,WAAW,UAAU,GAAG,QAAQ,CAClC;IACA;GAEF,KAAK;IACH,CAAC,KAAKG,iBAAiB,KAAKH,IAAI,QAAgB;oBACpC,kBAAkB;;SAE7B,GAAG,IAAI,GAAG,IAAI,QAAQ,GAAG,IAAI,MAAM,GAAG,IAAI,QAAQ,GAAG,IAAI,IAAI;IAC9D;GAEF,KAAK;IACH,CAAC,KAAKI,eAAe,KAAKJ,IAAI,QAAgB;yBAC7B,kBAAkB;;SAElC,GAAG,IAAI,GAAG,MAAM,QAAQ,GAAG,MAAM,IAAI;IACtC;GAEF,KAAK;IACH,CAAC,KAAKK,kBAAkB,KAAKL,IAAI,QAAgB;oBACrC,kBAAkB;;SAE7B,GAAG,IAAI,GAAG,KAAK,GAAG,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,GAAG;IACtD;GAEF,KAAK;IACH,KAAKM,cAAc,GAAG,OAAO,GAAG,MAAM;IACtC;GAEF,KAAK;IAIH,KAAK,MAAM,UAAU,GAAG,SACtB,KAAKA,cAAc,GAAG,OAAO,MAAM;IAErC;GAEF,SACE,YAAY,EAAE;EAClB;CACF;;;;;CAMA,YACE,OAC+C;EAC/C,MAAM,QAAQ,KAAKC,iBAAiB,KAAKP,IAAI,QAAgB;0CACvB,kBAAkB;;KAEvD,GAAG,IACF,MAAM,QACN,MAAM,IACR;EACA,OAAO,IAAI,IACT,KAAK,KAAK,EAAC,QAAQ,eAAc,CAC/B,QACA,aAAa,OACT,KAAA,IACC,WAAW,MAAM,QAAQ,CAChC,CAAC,CACH;CACF;;CAGA,YACE,OACA,QACA,UACM;EACN,CAAC,KAAKQ,iBAAiB,KAAKR,IAAI,QAAgB;gBACpC,kBAAkB;;KAE7B,GAAG,IACF,aAAa,KAAA,IAAY,OAAO,WAAW,UAAU,QAAQ,GAC7D,MAAM,QACN,MAAM,MACN,MACF;CACF;;;;;;CAOA,SAAS,OAAmB,SAAkC;EAC5D,KAAKC,KAAK;GAAC,IAAI;GAAqB;GAAO;EAAO,CAAC;CACrD;CAEA,cAAc,OAAmB,QAAsB;EACrD,CAAC,KAAKQ,gBAAgB,KAAKT,IAAI,QAAgB;qBAC9B,kBAAkB;;KAElC,GAAG,IAAI,MAAM,QAAQ,MAAM,MAAM,MAAM;CAC1C;AACF;;;;;;;;;AAUA,SAAgB,qBAAqB,IAAiC;CACpE,OAAO,qBAAqB,mBAAmB,EAAE,CAAC;AACpD;;;;;;;;AASA,SAAgB,qBAAqB,IAAiC;CACpE,MAAM,WAAW,qBAAqB,EAAE;CACxC,MAAM,WAAW,GACd,QAAgB;;gBAEL,kBAAkB;KAC7B,EACA,IAAuE;CAC1E,KAAK,MAAM,EAAC,QAAQ,OAAO,QAAQ,UAAU,UAAS,UAAU;EAI9D,MAAM,MAHM,SAAS,MACnB,MAAK,EAAE,MAAM,WAAW,UAAU,EAAE,MAAM,SAAS,KAEzC,GAAK,QAAQ;EACzB,IAAI,KACF,IAAI,WAAW,WAAW,MAAM,IAAI;CAExC;CACA,OAAO;AACT;;;;;;;;;;;;;;;;AAiBA,SAAgB,mBAAmB,IAAyB;CA6B1D,OAAO;EAAC,eA5Bc,GACnB,QAAgB;;;;;KAKhB,EACA,IAAuD,EACvD,KAAK,EAAC,QAAQ,OAAO,gBAAe;GACnC;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAgBM;EAAe,aAdH,GACjB,QAAgB;;gBAEL,kBAAkB;;KAE7B,EACA,IAAuE,EACvE,KAAK,EAAC,QAAQ,OAAO,QAAQ,gBAAe;GAC3C;GACA;GACA;GACA,UAAU,WAAW,MAAM,QAAQ;EACrC,EAEqB;CAAW;AACpC;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,sCACd,IACA,IACM;CACN,GAAG,QAAgB,gBAAgB,kBAAkB,EAAE,EAAE,IAAI;CAE7D,MAAM,cAAc,GACjB,QAAgB;;;;KAIhB,EACA,IAAiE;CAEpE,MAAM,eAAe,mBACnB,WACA,iCACA,sGAEE,oBACA,4EACJ;CACA,IAAI,YAAY,WAAW,GAAG;EAC5B,aAAa,IAAI,CAAC;EAClB;CACF;CAEA,MAAM,aAAa,IAAI,IACrB,GACG,QAAgB,qDAAqD,EACrE,IAAqC,EACrC,KAAK,EAAC,QAAQ,YAAW,CACxB,cAAc;EAAC;EAAQ,MAAM;CAAK,CAAC,GACnC;EAAC;EAAQ,MAAM;CAAK,CACtB,CAAC,CACL;CAEA,MAAM,SAAS,GAAG,QAAgB;mBACjB,kBAAkB;;GAElC;CACD,IAAI,aAAa;CACjB,KAAK,MAAM,EAAC,YAAY,UAAU,aAAa,cAAa,aAAa;EACvE,MAAM,QAAQ,WAAW,IAAI,QAAQ;EACrC,MAAM,MAAM,QAAQ,KAAK,SAAS,QAAQ,GAAG;EAC7C,MAAM,QAAQ,SAAS;GACrB,QAAQ,MAAM,IAAI,WAAW,SAAS,MAAM,GAAG,GAAG;GAClD,MAAM,MAAM,IAAI,WAAW,SAAS,MAAM,MAAM,CAAC;EACnD;EACA,IAAI,CAAC,SAAS,OAAO,GAAG;GACtB;GACA,GAAG,OACD,sCAAsC,SAAS,wBACvB,MAAM,OAAO,YAAY,MAAM,KAAK,GAC9D;EACF;EACA,OAAO,IAAI,MAAM,QAAQ,MAAM,MAAM,aAAa,QAAQ;CAC5D;CACA,aAAa,IAAI,UAAU;CAC3B,GAAG,OACD,WAAW,kBAAkB,SAAS,YAAY,OAAO,0BACvC,WAAW,gCAC/B;AACF"}

package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts

@@ -102,14 +102,14 @@ export declare class SQLiteChangeLogObserver {
     #private;
     constructor(lc: LogContext, info: SQLiteChangeLogInfo);
     /**
-     * Called as each message arrives, before it is written. Hashes the *received*
-     * stream, which is compared against the hash the writer computes over what it
-     * *stored*.
+     * Called as each message arrives, before it is written. For a sampled
+     * transaction, hashes the *received* stream and compares it with the hash the
+     * writer computes over what it *stored*.
      */
-    messageReceived(data: ChangeStreamData, json: string): void;
+    messageReceived(data: ChangeStreamData, json: string, validateHash?: boolean, storedChange?: string): void;
     /** Called after the message has been written to the log. */
-    messageProcessed(data: ChangeStreamData, commit: ChangeLogCommit | null, durationMs: number): void;
-    messageFailed(data: ChangeStreamData, error: unknown, durationMs: number): void;
+    messageProcessed(data: ChangeStreamData, commit: ChangeLogCommit | null, durationMs: number | undefined): void;
+    messageFailed(data: ChangeStreamData, error: unknown, durationMs: number | undefined): void;
     /**
      * A committed transaction folded schema changes into the cookie jar. Reported
      * separately from {@link messageProcessed} because it costs a read of the

package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-observability.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-observability.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAQ3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AAGtF,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,eAAe,EACrB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAIL,KAAK,aAAa,EAClB,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,+BAA+B,EAAC,MAAM,+BAA+B,CAAC;AAEnF;;;;;;;;;;;GAWG;AACH,wBAAgB,8BAA8B,CAC5C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,eAAe,GACtB,IAAI,CAsCN;AAED;;;;GAIG;AACH,MAAM,MAAM,0BAA0B,GAAG,aAAa,GAAG;IACvD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,0BAA0B,GAAG;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;CACtC,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,QAAQ,GAClB,0BAA0B,CAO5B;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,QAAQ,GAClB,mBAAmB,CAgBrB;AAED,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,0BAA0B,GAC/B,IAAI,CAcN;AAED;;;;;;;;GAQG;AACH,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAcjB;AAED,gEAAgE;AAChE,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,+BAA+B,CAAC;IAChD;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,iCAAiC,GAAG;IAC9C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B,CAAC;AAIF;;;;;GAKG;AACH,qBAAa,uBAAuB;;IAuElC,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,mBAAmB,EAoDpD;IAED;;;;OAIG;IACH,eAAe,CAAC,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAsD1D;IAED,4DAA4D;IAC5D,gBAAgB,CACd,IAAI,EAAE,gBAAgB,EACtB,MAAM,EAAE,eAAe,GAAG,IAAI,EAC9B,UAAU,EAAE,MAAM,GACjB,IAAI,CA+DN;IAED,aAAa,CACX,IAAI,EAAE,gBAAgB,EACtB,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,MAAM,GACjB,IAAI,CAoBN;IAED;;;;OAIG;IACH,cAAc,CAAC,GAAG,EAAE,SAAS,WAAW,EAAE,EAAE,IAAI,EAAE,eAAe,GAAG,IAAI,CAMvE;IAED;;;;;OAKG;IACH,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAEzE;IAED,yDAAyD;IACzD,KAAK,IAAI,IAAI,CAMZ;IAED,4EAA4E;IAC5E,UAAU,CAAC,IAAI,EAAE,mBAAmB,GAAG,IAAI,CAQ1C;IAED,KAAK,IAAI,iCAAiC,CAezC;CA2DF"}
+{"version":3,"file":"sqlite-change-log-observability.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-observability.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAQ3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AAGtF,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,eAAe,EACrB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAIL,KAAK,aAAa,EAClB,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,+BAA+B,EAAC,MAAM,+BAA+B,CAAC;AAEnF;;;;;;;;;;;GAWG;AACH,wBAAgB,8BAA8B,CAC5C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,eAAe,GACtB,IAAI,CAsCN;AAED;;;;GAIG;AACH,MAAM,MAAM,0BAA0B,GAAG,aAAa,GAAG;IACvD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,0BAA0B,GAAG;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;CACtC,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,QAAQ,GAClB,0BAA0B,CAO5B;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,QAAQ,GAClB,mBAAmB,CAgBrB;AAED,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,0BAA0B,GAC/B,IAAI,CAcN;AAED;;;;;;;;GAQG;AACH,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAcjB;AAED,gEAAgE;AAChE,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,+BAA+B,CAAC;IAChD;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,iCAAiC,GAAG;IAC9C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B,CAAC;AAIF;;;;;GAKG;AACH,qBAAa,uBAAuB;;IAuElC,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,mBAAmB,EAoDpD;IAED;;;;OAIG;IACH,eAAe,CACb,IAAI,EAAE,gBAAgB,EACtB,IAAI,EAAE,MAAM,EACZ,YAAY,UAAO,EACnB,YAAY,SAA4C,GACvD,IAAI,CA4DN;IAED,4DAA4D;IAC5D,gBAAgB,CACd,IAAI,EAAE,gBAAgB,EACtB,MAAM,EAAE,eAAe,GAAG,IAAI,EAC9B,UAAU,EAAE,MAAM,GAAG,SAAS,GAC7B,IAAI,CA0EN;IAED,aAAa,CACX,IAAI,EAAE,gBAAgB,EACtB,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,MAAM,GAAG,SAAS,GAC7B,IAAI,CAsBN;IAED;;;;OAIG;IACH,cAAc,CAAC,GAAG,EAAE,SAAS,WAAW,EAAE,EAAE,IAAI,EAAE,eAAe,GAAG,IAAI,CAMvE;IAED;;;;;OAKG;IACH,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAEzE;IAED,yDAAyD;IACzD,KAAK,IAAI,IAAI,CAMZ;IAED,4EAA4E;IAC5E,UAAU,CAAC,IAAI,EAAE,mBAAmB,GAAG,IAAI,CAQ1C;IAED,KAAK,IAAI,iCAAiC,CAezC;CA+DF"}

package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js

@@ -137,7 +137,7 @@ var TRANSACTION_ROW_BUCKETS = [
 */
 var SQLiteChangeLogObserver = class {
 	#lc;
-	#messageProcessing = getOrCreateLatencyHistogram("replica", "sqlite_change_log.message_processing_duration", "Time to write a replication message to the SQLite change log.");
+	#messageProcessing = getOrCreateLatencyHistogram("replica", "sqlite_change_log.message_processing_duration", "Sampled time to write a replication message to the SQLite change log.");
 	#logCommitDuration = getOrCreateLatencyHistogram("replica", "sqlite_change_log.log_commit_duration", "Time to append the commit row to the SQLite change log and commit its transaction. This is on the forward path: it precedes the forward of that transaction's commit message.");
 	#transactionRows = getOrCreateValueHistogram("replica", "sqlite_change_log.transaction_rows", {
 		description: "Rows stored per committed SQLite change-log transaction.",
@@ -146,7 +146,7 @@ var SQLiteChangeLogObserver = class {
 	});
 	#rollbackCounter = getOrCreateCounter("replica", "sqlite_change_log.rollbacks", "SQLite change-log transactions rolled back before commit.");
 	#invariantFailureCounter = getOrCreateCounter("replica", "sqlite_change_log.invariant_failures", "Detected SQLite change-log writer invariant failures.");
-	#hashComparisonCounter = getOrCreateCounter("replica", "sqlite_change_log.hash_comparisons", "Ephemeral transaction hash comparisons between the received change stream and SQLite change-log writes.");
+	#hashComparisonCounter = getOrCreateCounter("replica", "sqlite_change_log.hash_comparisons", "Sampled ephemeral transaction hash comparisons between the received change stream and SQLite change-log writes.");
 	#hashUnpairedCounter = getOrCreateCounter("replica", "sqlite_change_log.hash_unpaired", "Committed change-log transactions without both ephemeral hashes available for comparison.");
 	#cookieMutationCounter = getOrCreateCounter("replica", "sqlite_change_log.cookie_mutations", "Change-log cookie mutations folded from schema changes, by operation.");
 	#receivedHead;
@@ -190,23 +190,23 @@ var SQLiteChangeLogObserver = class {
 		});
 	}
 	/**
-	* Called as each message arrives, before it is written. Hashes the *received*
-	* stream, which is compared against the hash the writer computes over what it
-	* *stored*.
+	* Called as each message arrives, before it is written. For a sampled
+	* transaction, hashes the *received* stream and compares it with the hash the
+	* writer computes over what it *stored*.
 	*/
-	messageReceived(data, json) {
+	messageReceived(data, json, validateHash = true, storedChange = extractChangeSubstring(json, data[1].tag)) {
 		const tag = data[1].tag;
 		if (data[0] === "begin") {
 			if (this.#sourceHasher !== void 0) this.#invariantFailure("Received change-log begin while a source hash is already open", { receivedWatermark: data[2].commitWatermark });
 			const watermark = data[2].commitWatermark;
-			this.#sourceHasher = new ChangeLogTransactionHasher();
+			this.#sourceHasher = validateHash ? new ChangeLogTransactionHasher() : void 0;
 			this.#sourcePos = 0;
 			this.#sourceCommitHash = void 0;
-			this.#sourceHasher.add({
+			this.#sourceHasher?.add({
 				watermark,
 				pos: this.#sourcePos,
 				tag,
-				change: extractChangeSubstring(json, tag),
+				change: storedChange,
 				precommit: null
 			});
 			return;
@@ -216,6 +216,7 @@ var SQLiteChangeLogObserver = class {
 			return;
 		}
 		if (data[0] === "commit") this.#receivedHead = data[2].watermark;
+		if (!validateHash) return;
 		const watermark = this.#transactionWatermark;
 		if (this.#sourceHasher === void 0 || watermark === void 0) {
 			this.#invariantFailure("Received change-log message without an observed transaction", { tag });
@@ -226,7 +227,7 @@ var SQLiteChangeLogObserver = class {
 			watermark: commitWatermark,
 			pos: ++this.#sourcePos,
 			tag,
-			change: extractChangeSubstring(json, tag),
+			change: storedChange,
 			precommit: data[0] === "commit" ? watermark : null
 		});
 		if (data[0] === "commit") {
@@ -237,7 +238,7 @@ var SQLiteChangeLogObserver = class {
 	/** Called after the message has been written to the log. */
 	messageProcessed(data, commit, durationMs) {
 		const tag = data[1].tag;
-		this.#messageProcessing.recordMs(durationMs, {
+		if (durationMs !== void 0) this.#messageProcessing.recordMs(durationMs, {
 			tag,
 			outcome: "success"
 		});
@@ -273,7 +274,7 @@ var SQLiteChangeLogObserver = class {
 			this.#rows += commit.stats.rows;
 			this.#estimatedBytes += commit.stats.estimatedBytes;
 			this.#transactionRows.record(commit.stats.rows);
-			this.#compareHashes(watermark, this.#sourceCommitHash, commit.stats.hash);
+			if (this.#sourceCommitHash !== void 0 || commit.stats.hash !== void 0) this.#compareHashes(watermark, this.#sourceCommitHash, commit.stats.hash);
 		}
 		if (watermark < this.#sqliteHead) this.#invariantFailure("SQLite change-log head regressed", {
 			previousHead: this.#sqliteHead,
@@ -285,11 +286,11 @@ var SQLiteChangeLogObserver = class {
 	}
 	messageFailed(data, error, durationMs) {
 		const tag = data[1].tag;
-		this.#messageProcessing.recordMs(durationMs, {
+		if (durationMs !== void 0) this.#messageProcessing.recordMs(durationMs, {
 			tag,
 			outcome: "error"
 		});
-		if (data[0] === "commit") this.#recordUnpaired(data[2].watermark, "sqlite-commit-failed");
+		if (data[0] === "commit" && this.#sourceCommitHash !== void 0) this.#recordUnpaired(data[2].watermark, "sqlite-commit-failed");
 		if (error instanceof Error && error.name === "ChangeLogStreamInvariantError") this.#invariantFailure(error.message);
 		if (this.#transactionWatermark !== void 0 || data[0] === "begin") this.#recordRollback("processing-error");
 		this.#transactionWatermark = void 0;
@@ -364,6 +365,10 @@ var SQLiteChangeLogObserver = class {
 			this.#recordUnpaired(watermark, "missing-source-hash");
 			return;
 		}
+		if (sqliteHash === void 0) {
+			this.#recordUnpaired(watermark, "missing-sqlite-hash");
+			return;
+		}
 		if (sourceHash === sqliteHash) {
 			this.#hashMatches++;
 			this.#hashComparisonCounter.add(1, { outcome: "match" });

package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-observability.js","names":["#lc","#messageProcessing","#logCommitDuration","#transactionRows","#rollbackCounter","#invariantFailureCounter","#hashComparisonCounter","#hashUnpairedCounter","#cookieMutationCounter","#receivedHead","#sqliteHead","#rows","#estimatedBytes","#cookieRows","#sourceHasher","#invariantFailure","#sourcePos","#sourceCommitHash","#resetSourceHash","#transactionWatermark","#recordRollback","#recordUnpaired","#compareHashes","#cookieMutations","#rollbacks","#invariantFailures","#hashMatches","#hashMismatches","#hashUnpaired"],"sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-observability.ts"],"sourcesContent":["/**\n * Observability for the SQLite change log.\n *\n * Everything here now runs in the **change-streamer** process, which is where\n * the writer lives. The module stays beside the rest of the change-log code\n * under `services/replicator/` for the same reason `change-log-db.ts` and\n * `change-log-stream-writer.ts` do: moving files would obscure the topology\n * diff, and the writer's home is a property of who calls it rather than of\n * where it sits. Metric names are unchanged from 7H — only their emitting\n * process, and therefore their resource attributes, moved.\n */\n\nimport {stat} from 'node:fs/promises';\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n  getOrCreateValueHistogram,\n} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {\n  readCookieRowCounts,\n  type CookieOpTag,\n  type CookieRowCounts,\n} from './change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  readChangeLogBounds,\n  readChangeLogMeta,\n  type ChangeLogMeta,\n  type ReconcileResult,\n} from './change-log-db.ts';\nimport type {ChangeLogStreamTransactionStats} from './change-log-stream-writer.ts';\n\n/**\n * Reports the reconciliation of the change-log database against the watermark\n * the stream connection resumes from. This runs once per stream connection, so\n * a reconnect storm is visible here as well as a restart.\n *\n * A wipe is not an error — it is how every abnormal change-log state is\n * collapsed into a correct one — but each wipe costs the retention window, so\n * every reconnecting subscriber below the head gets `too-old`. The `gap` reason\n * in particular means the log neither held the resume watermark nor could be\n * truncated to it, so a nonzero rate in steady state is an alert, not a\n * statistic.\n */\nexport function recordSQLiteChangeLogReconcile(\n  lc: LogContext,\n  result: ReconcileResult,\n): void {\n  switch (result.action) {\n    case 'truncated':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_truncated_rows',\n        'Phantom change-log rows removed at reconciliation, i.e. rows above ' +\n          'the watermark the change stream resumed from.',\n      ).add(result.rows);\n      break;\n    case 'reseeded':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_wipes',\n        'Change-log wipes at reconciliation, by reason. Each one resets the ' +\n          'retention window, so subscribers below the head get `too-old`.',\n      ).add(1, {reason: result.reason});\n      break;\n    case 'none':\n      break;\n    default:\n      unreachable(result);\n  }\n  if (result.cookiesStale) {\n    // The invariant-17 counter: after any reconcile that moved the head, the\n    // log's cookie set has to be replaced with the one that belongs to the\n    // resume watermark. A nonzero `truncated` series in steady state means\n    // phantom truncation is happening on a shard where it should not.\n    getOrCreateCounter(\n      'replica',\n      'sqlite_change_log.cookie_reseed',\n      'Change-log cookie sets invalidated at reconciliation, by the ' +\n        'reconcile action that invalidated them.',\n    ).add(1, {action: result.action});\n  }\n  lc.debug?.('SQLite change-log reconciliation', {\n    sqliteChangeLogReconcile: result,\n  });\n}\n\n/**\n * What the log says about itself. Nothing here reads the replica: after the\n * writer moved to the change-streamer, the replica's state version is a\n * consequence of this log rather than a fact about it.\n */\nexport type SQLiteChangeLogStartupInfo = ChangeLogMeta & {\n  readonly minWatermark: string;\n  readonly headWatermark: string;\n};\n\nexport type SQLiteChangeLogInfo = SQLiteChangeLogStartupInfo & {\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n};\n\n/**\n * Reads the meta row and covered range. Unlike {@link getSQLiteChangeLogInfo}\n * this does not scan the log; see {@link readChangeLogBounds}.\n *\n * Call this after reconciliation, which is the only point at which the head is\n * known to be the resume watermark.\n */\nexport function getSQLiteChangeLogStartupInfo(\n  changeLog: Database,\n): SQLiteChangeLogStartupInfo {\n  const meta = readChangeLogMeta(changeLog);\n  const {minWatermark, headWatermark} = readChangeLogBounds(changeLog);\n  if (minWatermark === null || headWatermark === null) {\n    throw new Error('the SQLite change log must contain its seed transaction');\n  }\n  return {...meta, minWatermark, headWatermark};\n}\n\n/**\n * Extends {@link getSQLiteChangeLogStartupInfo} with the retained row and\n * estimated byte totals. Computing these scans the entire change-log table, so\n * only call it when the totals are consumed, i.e. when the writer (and thus the\n * observer) is enabled.\n */\nexport function getSQLiteChangeLogInfo(\n  changeLog: Database,\n): SQLiteChangeLogInfo {\n  const startup = getSQLiteChangeLogStartupInfo(changeLog);\n  const aggregate = changeLog\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\",\n             coalesce(sum(\"estimatedBytes\"), 0) AS \"estimatedBytes\"\n        FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    `)\n    .get<{rows: number; estimatedBytes: number}>();\n\n  return {\n    ...startup,\n    rows: aggregate.rows,\n    estimatedBytes: aggregate.estimatedBytes,\n    cookieRows: readCookieRowCounts(changeLog),\n  };\n}\n\nexport function logSQLiteChangeLogStartup(\n  lc: LogContext,\n  file: string,\n  info: SQLiteChangeLogStartupInfo,\n): void {\n  lc.info?.('SQLite change-log startup', {\n    sqliteChangeLog: {\n      file,\n      epoch: info.epoch,\n      generation: info.generation,\n      replicaID: info.replicaID,\n      schemaVersion: info.schemaVersion,\n      seedWatermark: info.seedWatermark,\n      seededAtMs: info.seededAtMs,\n      minWatermark: info.minWatermark,\n      headWatermark: info.headWatermark,\n    },\n  });\n}\n\n/**\n * The on-disk footprint of a SQLite database: its main file plus its `-wal` and\n * `-wal2` sidecars, whichever exist. `-shm` is excluded — it is a\n * shared-memory index rather than retained bytes.\n *\n * A missing file counts as zero rather than warning. Which sidecars exist\n * depends on the journal mode, and the change-log database is absent entirely\n * until the writer creates it.\n */\nexport async function sqliteFileBytes(\n  lc: LogContext,\n  file: string,\n): Promise<number> {\n  const sizes = await Promise.all(\n    [file, `${file}-wal`, `${file}-wal2`].map(async path => {\n      try {\n        return (await stat(path)).size;\n      } catch (e) {\n        if ((e as NodeJS.ErrnoException).code !== 'ENOENT') {\n          lc.warn?.(`unable to stat ${path} for size metrics`, e);\n        }\n        return 0;\n      }\n    }),\n  );\n  return sizes.reduce((total, size) => total + size, 0);\n}\n\n/** What the writer reports back for a committed transaction. */\nexport type ChangeLogCommit = {\n  readonly watermark: string;\n  readonly stats: ChangeLogStreamTransactionStats;\n  /**\n   * Appending the commit row and committing the log's transaction. This is the\n   * cost that sits on the forward path, once per upstream transaction.\n   */\n  readonly logCommitMs: number;\n};\n\nexport type SQLiteChangeLogObservabilityState = {\n  readonly receivedHead: string;\n  readonly sqliteHead: string;\n  readonly headLag: number | undefined;\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n  readonly cookieMutations: number;\n  readonly rollbacks: number;\n  readonly invariantFailures: number;\n  readonly hashMatches: number;\n  readonly hashMismatches: number;\n  readonly hashUnpaired: number;\n};\n\nconst TRANSACTION_ROW_BUCKETS = [2, 3, 5, 10, 25, 50, 100, 250, 1000, 5000];\n\n/**\n * Records writer metrics in the change-streamer process. State changes only\n * after the corresponding write succeeds, except receivedHead, which advances\n * when a commit message arrives so that transient lag behind the log is\n * observable.\n */\nexport class SQLiteChangeLogObserver {\n  readonly #lc: LogContext;\n  readonly #messageProcessing = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.message_processing_duration',\n    'Time to write a replication message to the SQLite change log.',\n  );\n  // The commit that sits on the forward path. There is no second commit to\n  // order against any more: the replica's belongs to a subscriber downstream of\n  // this loop, so `sqlite_change_log.replica_commit_duration` and the phantom\n  // window it measured are gone. A failed commit is still timed, by\n  // `message_processing_duration` keyed `tag=\"commit\"`.\n  readonly #logCommitDuration = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.log_commit_duration',\n    'Time to append the commit row to the SQLite change log and commit its ' +\n      'transaction. This is on the forward path: it precedes the forward of ' +\n      \"that transaction's commit message.\",\n  );\n  readonly #transactionRows = getOrCreateValueHistogram(\n    'replica',\n    'sqlite_change_log.transaction_rows',\n    {\n      description: 'Rows stored per committed SQLite change-log transaction.',\n      unit: '{row}',\n      bucketBoundaries: TRANSACTION_ROW_BUCKETS,\n    },\n  );\n  readonly #rollbackCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.rollbacks',\n    'SQLite change-log transactions rolled back before commit.',\n  );\n  readonly #invariantFailureCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.invariant_failures',\n    'Detected SQLite change-log writer invariant failures.',\n  );\n  readonly #hashComparisonCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_comparisons',\n    'Ephemeral transaction hash comparisons between the received change stream and SQLite change-log writes.',\n  );\n  readonly #hashUnpairedCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_unpaired',\n    'Committed change-log transactions without both ephemeral hashes available for comparison.',\n  );\n  // Near zero at steady state: cookie state only moves on schema changes.\n  readonly #cookieMutationCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.cookie_mutations',\n    'Change-log cookie mutations folded from schema changes, by operation.',\n  );\n\n  #receivedHead: string;\n  #sqliteHead: string;\n  #rows: number;\n  #estimatedBytes: number;\n  #cookieRows: CookieRowCounts;\n  #cookieMutations = 0;\n  #transactionWatermark: string | undefined;\n  #sourceHasher: ChangeLogTransactionHasher | undefined;\n  #sourcePos = 0;\n  #sourceCommitHash: string | undefined;\n  #rollbacks = 0;\n  #invariantFailures = 0;\n  #hashMatches = 0;\n  #hashMismatches = 0;\n  #hashUnpaired = 0;\n\n  constructor(lc: LogContext, info: SQLiteChangeLogInfo) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-observer');\n    // Reconciliation has just put the head at the resume watermark, which is\n    // also the point the stream is about to be received from.\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.rows',\n      'Rows retained in the SQLite change log.',\n    ).addCallback(result => result.observe(this.#rows));\n    // Nonzero with no backfill in flight is a leaked cookie, i.e. a backfill\n    // that will be re-requested forever.\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.cookie_rows',\n      'Cookie rows retained in the SQLite change log, by cookie table.',\n    ).addCallback(result => {\n      result.observe(this.#cookieRows.tableMetadata, {table: 'tableMetadata'});\n      result.observe(this.#cookieRows.backfilling, {table: 'backfilling'});\n    });\n    getOrCreateGauge('replica', 'sqlite_change_log.retained_bytes', {\n      description:\n        'Estimated UTF-8 payload bytes retained in the SQLite change log.',\n      unit: 'By',\n    }).addCallback(result => result.observe(this.#estimatedBytes));\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head',\n      'SQLite change-log head converted from its lexicographic watermark.',\n    ).addCallback(result => {\n      const head = watermarkValue(this.#sqliteHead);\n      if (head !== undefined) {\n        result.observe(head);\n      }\n    });\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head_lag',\n      'Distance from the latest received commit to the SQLite change-log ' +\n        'head. The log commits before the forward, so this is zero except ' +\n        'within a transaction.',\n    ).addCallback(result => {\n      const lag = watermarkDistance(this.#receivedHead, this.#sqliteHead);\n      if (lag !== undefined) {\n        result.observe(lag);\n      }\n    });\n  }\n\n  /**\n   * Called as each message arrives, before it is written. Hashes the *received*\n   * stream, which is compared against the hash the writer computes over what it\n   * *stored*.\n   */\n  messageReceived(data: ChangeStreamData, json: string): void {\n    const tag = data[1].tag;\n    if (data[0] === 'begin') {\n      if (this.#sourceHasher !== undefined) {\n        this.#invariantFailure(\n          'Received change-log begin while a source hash is already open',\n          {receivedWatermark: data[2].commitWatermark},\n        );\n      }\n      const watermark = data[2].commitWatermark;\n      this.#sourceHasher = new ChangeLogTransactionHasher();\n      this.#sourcePos = 0;\n      this.#sourceCommitHash = undefined;\n      this.#sourceHasher.add({\n        watermark,\n        pos: this.#sourcePos,\n        tag,\n        change: extractChangeSubstring(json, tag),\n        precommit: null,\n      });\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] === 'commit') {\n      this.#receivedHead = data[2].watermark;\n    }\n\n    const watermark = this.#transactionWatermark;\n    if (this.#sourceHasher === undefined || watermark === undefined) {\n      this.#invariantFailure(\n        'Received change-log message without an observed transaction',\n        {tag},\n      );\n      return;\n    }\n\n    const commitWatermark =\n      data[0] === 'commit' ? data[2].watermark : watermark;\n    this.#sourceHasher.add({\n      watermark: commitWatermark,\n      pos: ++this.#sourcePos,\n      tag,\n      change: extractChangeSubstring(json, tag),\n      precommit: data[0] === 'commit' ? watermark : null,\n    });\n    if (data[0] === 'commit') {\n      this.#sourceCommitHash = this.#sourceHasher.digest();\n      this.#sourceHasher = undefined;\n    }\n  }\n\n  /** Called after the message has been written to the log. */\n  messageProcessed(\n    data: ChangeStreamData,\n    commit: ChangeLogCommit | null,\n    durationMs: number,\n  ): void {\n    const tag = data[1].tag;\n    this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'success'});\n\n    if (data[0] === 'begin') {\n      if (this.#transactionWatermark !== undefined) {\n        this.#invariantFailure('SQLite change-log transaction already open', {\n          openWatermark: this.#transactionWatermark,\n          receivedWatermark: data[2].commitWatermark,\n        });\n      }\n      this.#transactionWatermark = data[2].commitWatermark;\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#recordRollback('upstream');\n      this.#transactionWatermark = undefined;\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] !== 'commit') {\n      return;\n    }\n\n    const watermark = data[2].watermark;\n    if (commit !== null) {\n      this.#logCommitDuration.recordMs(commit.logCommitMs);\n    }\n    if (this.#transactionWatermark !== watermark) {\n      this.#invariantFailure(\n        'SQLite change-log commit does not match the observed begin',\n        {openWatermark: this.#transactionWatermark, commitWatermark: watermark},\n      );\n    }\n    if (commit === null) {\n      this.#invariantFailure(\n        'SQLite change-log commit result is missing writer statistics',\n        {commitWatermark: watermark},\n      );\n      this.#recordUnpaired(watermark, 'missing-sqlite-hash');\n    } else {\n      if (commit.watermark !== watermark) {\n        this.#invariantFailure(\n          'SQLite change-log commit result has an unexpected watermark',\n          {commitWatermark: watermark, resultWatermark: commit.watermark},\n        );\n      }\n      this.#rows += commit.stats.rows;\n      this.#estimatedBytes += commit.stats.estimatedBytes;\n      this.#transactionRows.record(commit.stats.rows);\n      this.#compareHashes(watermark, this.#sourceCommitHash, commit.stats.hash);\n    }\n    if (watermark < this.#sqliteHead) {\n      this.#invariantFailure('SQLite change-log head regressed', {\n        previousHead: this.#sqliteHead,\n        commitWatermark: watermark,\n      });\n    }\n    this.#sqliteHead = watermark;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  messageFailed(\n    data: ChangeStreamData,\n    error: unknown,\n    durationMs: number,\n  ): void {\n    const tag = data[1].tag;\n    this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'error'});\n    if (data[0] === 'commit') {\n      // No commit duration here: a failed commit produces no statistics, so\n      // there is nothing to attribute. The attempt is timed by\n      // `message_processing_duration{tag=\"commit\",outcome=\"error\"}` above.\n      this.#recordUnpaired(data[2].watermark, 'sqlite-commit-failed');\n    }\n    if (\n      error instanceof Error &&\n      error.name === 'ChangeLogStreamInvariantError'\n    ) {\n      this.#invariantFailure(error.message);\n    }\n    if (this.#transactionWatermark !== undefined || data[0] === 'begin') {\n      this.#recordRollback('processing-error');\n    }\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  /**\n   * A committed transaction folded schema changes into the cookie jar. Reported\n   * separately from {@link messageProcessed} because it costs a read of the\n   * cookie tables, which is only worth taking when they actually moved.\n   */\n  cookiesMutated(ops: readonly CookieOpTag[], rows: CookieRowCounts): void {\n    for (const op of ops) {\n      this.#cookieMutations++;\n      this.#cookieMutationCounter.add(1, {op});\n    }\n    this.#cookieRows = rows;\n  }\n\n  /**\n   * A missed truncate-above surfaces as a constraint violation on the writer's\n   * plain `INSERT`, which the fail-soft policy would otherwise absorb quietly.\n   * Classifying it as an invariant failure is what keeps a reconciliation bug\n   * from looking like a lost cache.\n   */\n  invariantFailure(message: string, details?: Record<string, unknown>): void {\n    this.#invariantFailure(message, details);\n  }\n\n  /** The source interrupted the stream mid-transaction. */\n  abort(): void {\n    if (this.#transactionWatermark !== undefined) {\n      this.#recordRollback('source-interruption');\n      this.#transactionWatermark = undefined;\n    }\n    this.#resetSourceHash();\n  }\n\n  /** Reconciliation moved the head, so the observer's totals are restated. */\n  reconciled(info: SQLiteChangeLogInfo): void {\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  state(): SQLiteChangeLogObservabilityState {\n    return {\n      receivedHead: this.#receivedHead,\n      sqliteHead: this.#sqliteHead,\n      headLag: watermarkDistance(this.#receivedHead, this.#sqliteHead),\n      rows: this.#rows,\n      estimatedBytes: this.#estimatedBytes,\n      cookieRows: this.#cookieRows,\n      cookieMutations: this.#cookieMutations,\n      rollbacks: this.#rollbacks,\n      invariantFailures: this.#invariantFailures,\n      hashMatches: this.#hashMatches,\n      hashMismatches: this.#hashMismatches,\n      hashUnpaired: this.#hashUnpaired,\n    };\n  }\n\n  #recordRollback(\n    reason: 'upstream' | 'processing-error' | 'source-interruption',\n  ) {\n    this.#rollbacks++;\n    this.#rollbackCounter.add(1, {reason});\n  }\n\n  #invariantFailure(message: string, details?: Record<string, unknown>) {\n    this.#invariantFailures++;\n    this.#invariantFailureCounter.add(1);\n    this.#lc.error?.(message, details);\n  }\n\n  #compareHashes(\n    watermark: string,\n    sourceHash: string | undefined,\n    sqliteHash: string,\n  ): void {\n    if (sourceHash === undefined) {\n      this.#recordUnpaired(watermark, 'missing-source-hash');\n      return;\n    }\n    if (sourceHash === sqliteHash) {\n      this.#hashMatches++;\n      this.#hashComparisonCounter.add(1, {outcome: 'match'});\n      return;\n    }\n    this.#hashMismatches++;\n    this.#hashComparisonCounter.add(1, {outcome: 'mismatch'});\n    this.#lc.error?.('SQLite change-log transaction hash mismatch', {\n      sqliteChangeLogHashComparison: {\n        watermark,\n        sourceHash: sourceHash.slice(0, 16),\n        sqliteHash: sqliteHash.slice(0, 16),\n      },\n    });\n  }\n\n  #recordUnpaired(\n    watermark: string,\n    reason:\n      | 'missing-source-hash'\n      | 'missing-sqlite-hash'\n      | 'sqlite-commit-failed',\n  ): void {\n    this.#hashUnpaired++;\n    this.#hashUnpairedCounter.add(1, {reason});\n    this.#lc.warn?.('SQLite change-log transaction hash was not compared', {\n      sqliteChangeLogHashComparison: {watermark, reason},\n    });\n  }\n\n  #resetSourceHash(): void {\n    this.#sourceHasher = undefined;\n    this.#sourcePos = 0;\n    this.#sourceCommitHash = undefined;\n  }\n}\n\nfunction watermarkValue(watermark: string): number | undefined {\n  try {\n    const value = Number(versionFromLexi(watermark));\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n\nfunction watermarkDistance(\n  receivedHead: string,\n  sqliteHead: string,\n): number | undefined {\n  if (receivedHead <= sqliteHead) {\n    return 0;\n  }\n  try {\n    const distance =\n      versionFromLexi(receivedHead) - versionFromLexi(sqliteHead);\n    const value = Number(distance);\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA,SAAgB,+BACd,IACA,QACM;CACN,QAAQ,OAAO,QAAf;EACE,KAAK;GACH,mBACE,WACA,8CACA,kHAEF,EAAE,IAAI,OAAO,IAAI;GACjB;EACF,KAAK;GACH,mBACE,WACA,qCACA,mIAEF,EAAE,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;GAChC;EACF,KAAK,QACH;EACF,SACE,YAAY,MAAM;CACtB;CACA,IAAI,OAAO,cAKT,mBACE,WACA,mCACA,sGAEF,EAAE,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;CAElC,GAAG,QAAQ,oCAAoC,EAC7C,0BAA0B,OAC5B,CAAC;AACH;;;;;;;;AAyBA,SAAgB,8BACd,WAC4B;CAC5B,MAAM,OAAO,kBAAkB,SAAS;CACxC,MAAM,EAAC,cAAc,kBAAiB,oBAAoB,SAAS;CACnE,IAAI,iBAAiB,QAAQ,kBAAkB,MAC7C,MAAM,IAAI,MAAM,yDAAyD;CAE3E,OAAO;EAAC,GAAG;EAAM;EAAc;CAAa;AAC9C;;;;;;;AAQA,SAAgB,uBACd,WACqB;CACrB,MAAM,UAAU,8BAA8B,SAAS;CACvD,MAAM,YAAY,UACf,QAAgB;;;gBAGL,wBAAwB;KACnC,EACA,IAA4C;CAE/C,OAAO;EACL,GAAG;EACH,MAAM,UAAU;EAChB,gBAAgB,UAAU;EAC1B,YAAY,oBAAoB,SAAS;CAC3C;AACF;AAEA,SAAgB,0BACd,IACA,MACA,MACM;CACN,GAAG,OAAO,6BAA6B,EACrC,iBAAiB;EACf;EACA,OAAO,KAAK;EACZ,YAAY,KAAK;EACjB,WAAW,KAAK;EAChB,eAAe,KAAK;EACpB,eAAe,KAAK;EACpB,YAAY,KAAK;EACjB,cAAc,KAAK;EACnB,eAAe,KAAK;CACtB,EACF,CAAC;AACH;;;;;;;;;;AAWA,eAAsB,gBACpB,IACA,MACiB;CAajB,QAAO,MAZa,QAAQ,IAC1B;EAAC;EAAM,GAAG,KAAK;EAAO,GAAG,KAAK;CAAM,EAAE,IAAI,OAAM,SAAQ;EACtD,IAAI;GACF,QAAQ,MAAM,KAAK,IAAI,GAAG;EAC5B,SAAS,GAAG;GACV,IAAK,EAA4B,SAAS,UACxC,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;GAExD,OAAO;EACT;CACF,CAAC,CACH,GACa,QAAQ,OAAO,SAAS,QAAQ,MAAM,CAAC;AACtD;AA4BA,IAAM,0BAA0B;CAAC;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAM;AAAI;;;;;;;AAQ1E,IAAa,0BAAb,MAAqC;CACnC;CACA,qBAA8B,4BAC5B,WACA,iDACA,+DACF;CAMA,qBAA8B,4BAC5B,WACA,yCACA,+KAGF;CACA,mBAA4B,0BAC1B,WACA,sCACA;EACE,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CACF;CACA,mBAA4B,mBAC1B,WACA,+BACA,2DACF;CACA,2BAAoC,mBAClC,WACA,wCACA,uDACF;CACA,yBAAkC,mBAChC,WACA,sCACA,yGACF;CACA,uBAAgC,mBAC9B,WACA,mCACA,2FACF;CAEA,yBAAkC,mBAChC,WACA,sCACA,uEACF;CAEA;CACA;CACA;CACA;CACA;CACA,mBAAmB;CACnB;CACA;CACA,aAAa;CACb;CACA,aAAa;CACb,qBAAqB;CACrB,eAAe;CACf,kBAAkB;CAClB,gBAAgB;CAEhB,YAAY,IAAgB,MAA2B;EACrD,KAAKA,MAAM,GAAG,YAAY,aAAa,4BAA4B;EAGnE,KAAKS,gBAAgB,KAAK;EAC1B,KAAKC,cAAc,KAAK;EACxB,KAAKC,QAAQ,KAAK;EAClB,KAAKC,kBAAkB,KAAK;EAC5B,KAAKC,cAAc,KAAK;EAExB,iBACE,WACA,0BACA,yCACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKF,KAAK,CAAC;EAGlD,iBACE,WACA,iCACA,iEACF,EAAE,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAKE,YAAY,eAAe,EAAC,OAAO,gBAAe,CAAC;GACvE,OAAO,QAAQ,KAAKA,YAAY,aAAa,EAAC,OAAO,cAAa,CAAC;EACrE,CAAC;EACD,iBAAiB,WAAW,oCAAoC;GAC9D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKD,eAAe,CAAC;EAC7D,iBACE,WACA,0BACA,oEACF,EAAE,aAAY,WAAU;GACtB,MAAM,OAAO,eAAe,KAAKF,WAAW;GAC5C,IAAI,SAAS,KAAA,GACX,OAAO,QAAQ,IAAI;EAEvB,CAAC;EACD,iBACE,WACA,8BACA,0JAGF,EAAE,aAAY,WAAU;GACtB,MAAM,MAAM,kBAAkB,KAAKD,eAAe,KAAKC,WAAW;GAClE,IAAI,QAAQ,KAAA,GACV,OAAO,QAAQ,GAAG;EAEtB,CAAC;CACH;;;;;;CAOA,gBAAgB,MAAwB,MAAoB;EAC1D,MAAM,MAAM,KAAK,GAAG;EACpB,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAKI,kBAAkB,KAAA,GACzB,KAAKC,kBACH,iEACA,EAAC,mBAAmB,KAAK,GAAG,gBAAe,CAC7C;GAEF,MAAM,YAAY,KAAK,GAAG;GAC1B,KAAKD,gBAAgB,IAAI,2BAA2B;GACpD,KAAKE,aAAa;GAClB,KAAKC,oBAAoB,KAAA;GACzB,KAAKH,cAAc,IAAI;IACrB;IACA,KAAK,KAAKE;IACV;IACA,QAAQ,uBAAuB,MAAM,GAAG;IACxC,WAAW;GACb,CAAC;GACD;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAKE,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd,KAAKT,gBAAgB,KAAK,GAAG;EAG/B,MAAM,YAAY,KAAKU;EACvB,IAAI,KAAKL,kBAAkB,KAAA,KAAa,cAAc,KAAA,GAAW;GAC/D,KAAKC,kBACH,+DACA,EAAC,IAAG,CACN;GACA;EACF;EAEA,MAAM,kBACJ,KAAK,OAAO,WAAW,KAAK,GAAG,YAAY;EAC7C,KAAKD,cAAc,IAAI;GACrB,WAAW;GACX,KAAK,EAAE,KAAKE;GACZ;GACA,QAAQ,uBAAuB,MAAM,GAAG;GACxC,WAAW,KAAK,OAAO,WAAW,YAAY;EAChD,CAAC;EACD,IAAI,KAAK,OAAO,UAAU;GACxB,KAAKC,oBAAoB,KAAKH,cAAc,OAAO;GACnD,KAAKA,gBAAgB,KAAA;EACvB;CACF;;CAGA,iBACE,MACA,QACA,YACM;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,KAAKb,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAS,CAAC;EAEtE,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAKkB,0BAA0B,KAAA,GACjC,KAAKJ,kBAAkB,8CAA8C;IACnE,eAAe,KAAKI;IACpB,mBAAmB,KAAK,GAAG;GAC7B,CAAC;GAEH,KAAKA,wBAAwB,KAAK,GAAG;GACrC;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAKC,gBAAgB,UAAU;GAC/B,KAAKD,wBAAwB,KAAA;GAC7B,KAAKD,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd;EAGF,MAAM,YAAY,KAAK,GAAG;EAC1B,IAAI,WAAW,MACb,KAAKhB,mBAAmB,SAAS,OAAO,WAAW;EAErD,IAAI,KAAKiB,0BAA0B,WACjC,KAAKJ,kBACH,8DACA;GAAC,eAAe,KAAKI;GAAuB,iBAAiB;EAAS,CACxE;EAEF,IAAI,WAAW,MAAM;GACnB,KAAKJ,kBACH,gEACA,EAAC,iBAAiB,UAAS,CAC7B;GACA,KAAKM,gBAAgB,WAAW,qBAAqB;EACvD,OAAO;GACL,IAAI,OAAO,cAAc,WACvB,KAAKN,kBACH,+DACA;IAAC,iBAAiB;IAAW,iBAAiB,OAAO;GAAS,CAChE;GAEF,KAAKJ,SAAS,OAAO,MAAM;GAC3B,KAAKC,mBAAmB,OAAO,MAAM;GACrC,KAAKT,iBAAiB,OAAO,OAAO,MAAM,IAAI;GAC9C,KAAKmB,eAAe,WAAW,KAAKL,mBAAmB,OAAO,MAAM,IAAI;EAC1E;EACA,IAAI,YAAY,KAAKP,aACnB,KAAKK,kBAAkB,oCAAoC;GACzD,cAAc,KAAKL;GACnB,iBAAiB;EACnB,CAAC;EAEH,KAAKA,cAAc;EACnB,KAAKS,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;CAEA,cACE,MACA,OACA,YACM;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,KAAKjB,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAO,CAAC;EACpE,IAAI,KAAK,OAAO,UAId,KAAKoB,gBAAgB,KAAK,GAAG,WAAW,sBAAsB;EAEhE,IACE,iBAAiB,SACjB,MAAM,SAAS,iCAEf,KAAKN,kBAAkB,MAAM,OAAO;EAEtC,IAAI,KAAKI,0BAA0B,KAAA,KAAa,KAAK,OAAO,SAC1D,KAAKC,gBAAgB,kBAAkB;EAEzC,KAAKD,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;;;;;;CAOA,eAAe,KAA6B,MAA6B;EACvE,KAAK,MAAM,MAAM,KAAK;GACpB,KAAKK;GACL,KAAKf,uBAAuB,IAAI,GAAG,EAAC,GAAE,CAAC;EACzC;EACA,KAAKK,cAAc;CACrB;;;;;;;CAQA,iBAAiB,SAAiB,SAAyC;EACzE,KAAKE,kBAAkB,SAAS,OAAO;CACzC;;CAGA,QAAc;EACZ,IAAI,KAAKI,0BAA0B,KAAA,GAAW;GAC5C,KAAKC,gBAAgB,qBAAqB;GAC1C,KAAKD,wBAAwB,KAAA;EAC/B;EACA,KAAKD,iBAAiB;CACxB;;CAGA,WAAW,MAAiC;EAC1C,KAAKT,gBAAgB,KAAK;EAC1B,KAAKC,cAAc,KAAK;EACxB,KAAKC,QAAQ,KAAK;EAClB,KAAKC,kBAAkB,KAAK;EAC5B,KAAKC,cAAc,KAAK;EACxB,KAAKM,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;CAEA,QAA2C;EACzC,OAAO;GACL,cAAc,KAAKT;GACnB,YAAY,KAAKC;GACjB,SAAS,kBAAkB,KAAKD,eAAe,KAAKC,WAAW;GAC/D,MAAM,KAAKC;GACX,gBAAgB,KAAKC;GACrB,YAAY,KAAKC;GACjB,iBAAiB,KAAKU;GACtB,WAAW,KAAKC;GAChB,mBAAmB,KAAKC;GACxB,aAAa,KAAKC;GAClB,gBAAgB,KAAKC;GACrB,cAAc,KAAKC;EACrB;CACF;CAEA,gBACE,QACA;EACA,KAAKJ;EACL,KAAKpB,iBAAiB,IAAI,GAAG,EAAC,OAAM,CAAC;CACvC;CAEA,kBAAkB,SAAiB,SAAmC;EACpE,KAAKqB;EACL,KAAKpB,yBAAyB,IAAI,CAAC;EACnC,KAAKL,IAAI,QAAQ,SAAS,OAAO;CACnC;CAEA,eACE,WACA,YACA,YACM;EACN,IAAI,eAAe,KAAA,GAAW;GAC5B,KAAKqB,gBAAgB,WAAW,qBAAqB;GACrD;EACF;EACA,IAAI,eAAe,YAAY;GAC7B,KAAKK;GACL,KAAKpB,uBAAuB,IAAI,GAAG,EAAC,SAAS,QAAO,CAAC;GACrD;EACF;EACA,KAAKqB;EACL,KAAKrB,uBAAuB,IAAI,GAAG,EAAC,SAAS,WAAU,CAAC;EACxD,KAAKN,IAAI,QAAQ,+CAA+C,EAC9D,+BAA+B;GAC7B;GACA,YAAY,WAAW,MAAM,GAAG,EAAE;GAClC,YAAY,WAAW,MAAM,GAAG,EAAE;EACpC,EACF,CAAC;CACH;CAEA,gBACE,WACA,QAIM;EACN,KAAK4B;EACL,KAAKrB,qBAAqB,IAAI,GAAG,EAAC,OAAM,CAAC;EACzC,KAAKP,IAAI,OAAO,uDAAuD,EACrE,+BAA+B;GAAC;GAAW;EAAM,EACnD,CAAC;CACH;CAEA,mBAAyB;EACvB,KAAKc,gBAAgB,KAAA;EACrB,KAAKE,aAAa;EAClB,KAAKC,oBAAoB,KAAA;CAC3B;AACF;AAEA,SAAS,eAAe,WAAuC;CAC7D,IAAI;EACF,MAAM,QAAQ,OAAO,gBAAgB,SAAS,CAAC;EAC/C,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF;AAEA,SAAS,kBACP,cACA,YACoB;CACpB,IAAI,gBAAgB,YAClB,OAAO;CAET,IAAI;EACF,MAAM,WACJ,gBAAgB,YAAY,IAAI,gBAAgB,UAAU;EAC5D,MAAM,QAAQ,OAAO,QAAQ;EAC7B,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF"}
+{"version":3,"file":"sqlite-change-log-observability.js","names":["#lc","#messageProcessing","#logCommitDuration","#transactionRows","#rollbackCounter","#invariantFailureCounter","#hashComparisonCounter","#hashUnpairedCounter","#cookieMutationCounter","#receivedHead","#sqliteHead","#rows","#estimatedBytes","#cookieRows","#sourceHasher","#invariantFailure","#sourcePos","#sourceCommitHash","#resetSourceHash","#transactionWatermark","#recordRollback","#recordUnpaired","#compareHashes","#cookieMutations","#rollbacks","#invariantFailures","#hashMatches","#hashMismatches","#hashUnpaired"],"sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-observability.ts"],"sourcesContent":["/**\n * Observability for the SQLite change log.\n *\n * Everything here now runs in the **change-streamer** process, which is where\n * the writer lives. The module stays beside the rest of the change-log code\n * under `services/replicator/` for the same reason `change-log-db.ts` and\n * `change-log-stream-writer.ts` do: moving files would obscure the topology\n * diff, and the writer's home is a property of who calls it rather than of\n * where it sits. Metric names are unchanged from 7H — only their emitting\n * process, and therefore their resource attributes, moved.\n */\n\nimport {stat} from 'node:fs/promises';\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n  getOrCreateValueHistogram,\n} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {\n  readCookieRowCounts,\n  type CookieOpTag,\n  type CookieRowCounts,\n} from './change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  readChangeLogBounds,\n  readChangeLogMeta,\n  type ChangeLogMeta,\n  type ReconcileResult,\n} from './change-log-db.ts';\nimport type {ChangeLogStreamTransactionStats} from './change-log-stream-writer.ts';\n\n/**\n * Reports the reconciliation of the change-log database against the watermark\n * the stream connection resumes from. This runs once per stream connection, so\n * a reconnect storm is visible here as well as a restart.\n *\n * A wipe is not an error — it is how every abnormal change-log state is\n * collapsed into a correct one — but each wipe costs the retention window, so\n * every reconnecting subscriber below the head gets `too-old`. The `gap` reason\n * in particular means the log neither held the resume watermark nor could be\n * truncated to it, so a nonzero rate in steady state is an alert, not a\n * statistic.\n */\nexport function recordSQLiteChangeLogReconcile(\n  lc: LogContext,\n  result: ReconcileResult,\n): void {\n  switch (result.action) {\n    case 'truncated':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_truncated_rows',\n        'Phantom change-log rows removed at reconciliation, i.e. rows above ' +\n          'the watermark the change stream resumed from.',\n      ).add(result.rows);\n      break;\n    case 'reseeded':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_wipes',\n        'Change-log wipes at reconciliation, by reason. Each one resets the ' +\n          'retention window, so subscribers below the head get `too-old`.',\n      ).add(1, {reason: result.reason});\n      break;\n    case 'none':\n      break;\n    default:\n      unreachable(result);\n  }\n  if (result.cookiesStale) {\n    // The invariant-17 counter: after any reconcile that moved the head, the\n    // log's cookie set has to be replaced with the one that belongs to the\n    // resume watermark. A nonzero `truncated` series in steady state means\n    // phantom truncation is happening on a shard where it should not.\n    getOrCreateCounter(\n      'replica',\n      'sqlite_change_log.cookie_reseed',\n      'Change-log cookie sets invalidated at reconciliation, by the ' +\n        'reconcile action that invalidated them.',\n    ).add(1, {action: result.action});\n  }\n  lc.debug?.('SQLite change-log reconciliation', {\n    sqliteChangeLogReconcile: result,\n  });\n}\n\n/**\n * What the log says about itself. Nothing here reads the replica: after the\n * writer moved to the change-streamer, the replica's state version is a\n * consequence of this log rather than a fact about it.\n */\nexport type SQLiteChangeLogStartupInfo = ChangeLogMeta & {\n  readonly minWatermark: string;\n  readonly headWatermark: string;\n};\n\nexport type SQLiteChangeLogInfo = SQLiteChangeLogStartupInfo & {\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n};\n\n/**\n * Reads the meta row and covered range. Unlike {@link getSQLiteChangeLogInfo}\n * this does not scan the log; see {@link readChangeLogBounds}.\n *\n * Call this after reconciliation, which is the only point at which the head is\n * known to be the resume watermark.\n */\nexport function getSQLiteChangeLogStartupInfo(\n  changeLog: Database,\n): SQLiteChangeLogStartupInfo {\n  const meta = readChangeLogMeta(changeLog);\n  const {minWatermark, headWatermark} = readChangeLogBounds(changeLog);\n  if (minWatermark === null || headWatermark === null) {\n    throw new Error('the SQLite change log must contain its seed transaction');\n  }\n  return {...meta, minWatermark, headWatermark};\n}\n\n/**\n * Extends {@link getSQLiteChangeLogStartupInfo} with the retained row and\n * estimated byte totals. Computing these scans the entire change-log table, so\n * only call it when the totals are consumed, i.e. when the writer (and thus the\n * observer) is enabled.\n */\nexport function getSQLiteChangeLogInfo(\n  changeLog: Database,\n): SQLiteChangeLogInfo {\n  const startup = getSQLiteChangeLogStartupInfo(changeLog);\n  const aggregate = changeLog\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\",\n             coalesce(sum(\"estimatedBytes\"), 0) AS \"estimatedBytes\"\n        FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    `)\n    .get<{rows: number; estimatedBytes: number}>();\n\n  return {\n    ...startup,\n    rows: aggregate.rows,\n    estimatedBytes: aggregate.estimatedBytes,\n    cookieRows: readCookieRowCounts(changeLog),\n  };\n}\n\nexport function logSQLiteChangeLogStartup(\n  lc: LogContext,\n  file: string,\n  info: SQLiteChangeLogStartupInfo,\n): void {\n  lc.info?.('SQLite change-log startup', {\n    sqliteChangeLog: {\n      file,\n      epoch: info.epoch,\n      generation: info.generation,\n      replicaID: info.replicaID,\n      schemaVersion: info.schemaVersion,\n      seedWatermark: info.seedWatermark,\n      seededAtMs: info.seededAtMs,\n      minWatermark: info.minWatermark,\n      headWatermark: info.headWatermark,\n    },\n  });\n}\n\n/**\n * The on-disk footprint of a SQLite database: its main file plus its `-wal` and\n * `-wal2` sidecars, whichever exist. `-shm` is excluded — it is a\n * shared-memory index rather than retained bytes.\n *\n * A missing file counts as zero rather than warning. Which sidecars exist\n * depends on the journal mode, and the change-log database is absent entirely\n * until the writer creates it.\n */\nexport async function sqliteFileBytes(\n  lc: LogContext,\n  file: string,\n): Promise<number> {\n  const sizes = await Promise.all(\n    [file, `${file}-wal`, `${file}-wal2`].map(async path => {\n      try {\n        return (await stat(path)).size;\n      } catch (e) {\n        if ((e as NodeJS.ErrnoException).code !== 'ENOENT') {\n          lc.warn?.(`unable to stat ${path} for size metrics`, e);\n        }\n        return 0;\n      }\n    }),\n  );\n  return sizes.reduce((total, size) => total + size, 0);\n}\n\n/** What the writer reports back for a committed transaction. */\nexport type ChangeLogCommit = {\n  readonly watermark: string;\n  readonly stats: ChangeLogStreamTransactionStats;\n  /**\n   * Appending the commit row and committing the log's transaction. This is the\n   * cost that sits on the forward path, once per upstream transaction.\n   */\n  readonly logCommitMs: number;\n};\n\nexport type SQLiteChangeLogObservabilityState = {\n  readonly receivedHead: string;\n  readonly sqliteHead: string;\n  readonly headLag: number | undefined;\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n  readonly cookieMutations: number;\n  readonly rollbacks: number;\n  readonly invariantFailures: number;\n  readonly hashMatches: number;\n  readonly hashMismatches: number;\n  readonly hashUnpaired: number;\n};\n\nconst TRANSACTION_ROW_BUCKETS = [2, 3, 5, 10, 25, 50, 100, 250, 1000, 5000];\n\n/**\n * Records writer metrics in the change-streamer process. State changes only\n * after the corresponding write succeeds, except receivedHead, which advances\n * when a commit message arrives so that transient lag behind the log is\n * observable.\n */\nexport class SQLiteChangeLogObserver {\n  readonly #lc: LogContext;\n  readonly #messageProcessing = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.message_processing_duration',\n    'Sampled time to write a replication message to the SQLite change log.',\n  );\n  // The commit that sits on the forward path. There is no second commit to\n  // order against any more: the replica's belongs to a subscriber downstream of\n  // this loop, so `sqlite_change_log.replica_commit_duration` and the phantom\n  // window it measured are gone. Sampled failed commits are still timed by\n  // `message_processing_duration` keyed `tag=\"commit\"`.\n  readonly #logCommitDuration = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.log_commit_duration',\n    'Time to append the commit row to the SQLite change log and commit its ' +\n      'transaction. This is on the forward path: it precedes the forward of ' +\n      \"that transaction's commit message.\",\n  );\n  readonly #transactionRows = getOrCreateValueHistogram(\n    'replica',\n    'sqlite_change_log.transaction_rows',\n    {\n      description: 'Rows stored per committed SQLite change-log transaction.',\n      unit: '{row}',\n      bucketBoundaries: TRANSACTION_ROW_BUCKETS,\n    },\n  );\n  readonly #rollbackCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.rollbacks',\n    'SQLite change-log transactions rolled back before commit.',\n  );\n  readonly #invariantFailureCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.invariant_failures',\n    'Detected SQLite change-log writer invariant failures.',\n  );\n  readonly #hashComparisonCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_comparisons',\n    'Sampled ephemeral transaction hash comparisons between the received change stream and SQLite change-log writes.',\n  );\n  readonly #hashUnpairedCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_unpaired',\n    'Committed change-log transactions without both ephemeral hashes available for comparison.',\n  );\n  // Near zero at steady state: cookie state only moves on schema changes.\n  readonly #cookieMutationCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.cookie_mutations',\n    'Change-log cookie mutations folded from schema changes, by operation.',\n  );\n\n  #receivedHead: string;\n  #sqliteHead: string;\n  #rows: number;\n  #estimatedBytes: number;\n  #cookieRows: CookieRowCounts;\n  #cookieMutations = 0;\n  #transactionWatermark: string | undefined;\n  #sourceHasher: ChangeLogTransactionHasher | undefined;\n  #sourcePos = 0;\n  #sourceCommitHash: string | undefined;\n  #rollbacks = 0;\n  #invariantFailures = 0;\n  #hashMatches = 0;\n  #hashMismatches = 0;\n  #hashUnpaired = 0;\n\n  constructor(lc: LogContext, info: SQLiteChangeLogInfo) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-observer');\n    // Reconciliation has just put the head at the resume watermark, which is\n    // also the point the stream is about to be received from.\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.rows',\n      'Rows retained in the SQLite change log.',\n    ).addCallback(result => result.observe(this.#rows));\n    // Nonzero with no backfill in flight is a leaked cookie, i.e. a backfill\n    // that will be re-requested forever.\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.cookie_rows',\n      'Cookie rows retained in the SQLite change log, by cookie table.',\n    ).addCallback(result => {\n      result.observe(this.#cookieRows.tableMetadata, {table: 'tableMetadata'});\n      result.observe(this.#cookieRows.backfilling, {table: 'backfilling'});\n    });\n    getOrCreateGauge('replica', 'sqlite_change_log.retained_bytes', {\n      description:\n        'Estimated UTF-8 payload bytes retained in the SQLite change log.',\n      unit: 'By',\n    }).addCallback(result => result.observe(this.#estimatedBytes));\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head',\n      'SQLite change-log head converted from its lexicographic watermark.',\n    ).addCallback(result => {\n      const head = watermarkValue(this.#sqliteHead);\n      if (head !== undefined) {\n        result.observe(head);\n      }\n    });\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head_lag',\n      'Distance from the latest received commit to the SQLite change-log ' +\n        'head. The log commits before the forward, so this is zero except ' +\n        'within a transaction.',\n    ).addCallback(result => {\n      const lag = watermarkDistance(this.#receivedHead, this.#sqliteHead);\n      if (lag !== undefined) {\n        result.observe(lag);\n      }\n    });\n  }\n\n  /**\n   * Called as each message arrives, before it is written. For a sampled\n   * transaction, hashes the *received* stream and compares it with the hash the\n   * writer computes over what it *stored*.\n   */\n  messageReceived(\n    data: ChangeStreamData,\n    json: string,\n    validateHash = true,\n    storedChange = extractChangeSubstring(json, data[1].tag),\n  ): void {\n    const tag = data[1].tag;\n    if (data[0] === 'begin') {\n      if (this.#sourceHasher !== undefined) {\n        this.#invariantFailure(\n          'Received change-log begin while a source hash is already open',\n          {receivedWatermark: data[2].commitWatermark},\n        );\n      }\n      const watermark = data[2].commitWatermark;\n      this.#sourceHasher = validateHash\n        ? new ChangeLogTransactionHasher()\n        : undefined;\n      this.#sourcePos = 0;\n      this.#sourceCommitHash = undefined;\n      this.#sourceHasher?.add({\n        watermark,\n        pos: this.#sourcePos,\n        tag,\n        change: storedChange,\n        precommit: null,\n      });\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] === 'commit') {\n      this.#receivedHead = data[2].watermark;\n    }\n\n    if (!validateHash) {\n      return;\n    }\n\n    const watermark = this.#transactionWatermark;\n    if (this.#sourceHasher === undefined || watermark === undefined) {\n      this.#invariantFailure(\n        'Received change-log message without an observed transaction',\n        {tag},\n      );\n      return;\n    }\n\n    const commitWatermark =\n      data[0] === 'commit' ? data[2].watermark : watermark;\n    this.#sourceHasher.add({\n      watermark: commitWatermark,\n      pos: ++this.#sourcePos,\n      tag,\n      change: storedChange,\n      precommit: data[0] === 'commit' ? watermark : null,\n    });\n    if (data[0] === 'commit') {\n      this.#sourceCommitHash = this.#sourceHasher.digest();\n      this.#sourceHasher = undefined;\n    }\n  }\n\n  /** Called after the message has been written to the log. */\n  messageProcessed(\n    data: ChangeStreamData,\n    commit: ChangeLogCommit | null,\n    durationMs: number | undefined,\n  ): void {\n    const tag = data[1].tag;\n    if (durationMs !== undefined) {\n      this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'success'});\n    }\n\n    if (data[0] === 'begin') {\n      if (this.#transactionWatermark !== undefined) {\n        this.#invariantFailure('SQLite change-log transaction already open', {\n          openWatermark: this.#transactionWatermark,\n          receivedWatermark: data[2].commitWatermark,\n        });\n      }\n      this.#transactionWatermark = data[2].commitWatermark;\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#recordRollback('upstream');\n      this.#transactionWatermark = undefined;\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] !== 'commit') {\n      return;\n    }\n\n    const watermark = data[2].watermark;\n    if (commit !== null) {\n      this.#logCommitDuration.recordMs(commit.logCommitMs);\n    }\n    if (this.#transactionWatermark !== watermark) {\n      this.#invariantFailure(\n        'SQLite change-log commit does not match the observed begin',\n        {openWatermark: this.#transactionWatermark, commitWatermark: watermark},\n      );\n    }\n    if (commit === null) {\n      this.#invariantFailure(\n        'SQLite change-log commit result is missing writer statistics',\n        {commitWatermark: watermark},\n      );\n      this.#recordUnpaired(watermark, 'missing-sqlite-hash');\n    } else {\n      if (commit.watermark !== watermark) {\n        this.#invariantFailure(\n          'SQLite change-log commit result has an unexpected watermark',\n          {commitWatermark: watermark, resultWatermark: commit.watermark},\n        );\n      }\n      this.#rows += commit.stats.rows;\n      this.#estimatedBytes += commit.stats.estimatedBytes;\n      this.#transactionRows.record(commit.stats.rows);\n      if (\n        this.#sourceCommitHash !== undefined ||\n        commit.stats.hash !== undefined\n      ) {\n        this.#compareHashes(\n          watermark,\n          this.#sourceCommitHash,\n          commit.stats.hash,\n        );\n      }\n    }\n    if (watermark < this.#sqliteHead) {\n      this.#invariantFailure('SQLite change-log head regressed', {\n        previousHead: this.#sqliteHead,\n        commitWatermark: watermark,\n      });\n    }\n    this.#sqliteHead = watermark;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  messageFailed(\n    data: ChangeStreamData,\n    error: unknown,\n    durationMs: number | undefined,\n  ): void {\n    const tag = data[1].tag;\n    if (durationMs !== undefined) {\n      this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'error'});\n    }\n    if (data[0] === 'commit' && this.#sourceCommitHash !== undefined) {\n      // No commit duration here: a failed commit produces no statistics, so\n      // there is nothing to attribute. The attempt is timed by\n      // `message_processing_duration{tag=\"commit\",outcome=\"error\"}` above.\n      this.#recordUnpaired(data[2].watermark, 'sqlite-commit-failed');\n    }\n    if (\n      error instanceof Error &&\n      error.name === 'ChangeLogStreamInvariantError'\n    ) {\n      this.#invariantFailure(error.message);\n    }\n    if (this.#transactionWatermark !== undefined || data[0] === 'begin') {\n      this.#recordRollback('processing-error');\n    }\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  /**\n   * A committed transaction folded schema changes into the cookie jar. Reported\n   * separately from {@link messageProcessed} because it costs a read of the\n   * cookie tables, which is only worth taking when they actually moved.\n   */\n  cookiesMutated(ops: readonly CookieOpTag[], rows: CookieRowCounts): void {\n    for (const op of ops) {\n      this.#cookieMutations++;\n      this.#cookieMutationCounter.add(1, {op});\n    }\n    this.#cookieRows = rows;\n  }\n\n  /**\n   * A missed truncate-above surfaces as a constraint violation on the writer's\n   * plain `INSERT`, which the fail-soft policy would otherwise absorb quietly.\n   * Classifying it as an invariant failure is what keeps a reconciliation bug\n   * from looking like a lost cache.\n   */\n  invariantFailure(message: string, details?: Record<string, unknown>): void {\n    this.#invariantFailure(message, details);\n  }\n\n  /** The source interrupted the stream mid-transaction. */\n  abort(): void {\n    if (this.#transactionWatermark !== undefined) {\n      this.#recordRollback('source-interruption');\n      this.#transactionWatermark = undefined;\n    }\n    this.#resetSourceHash();\n  }\n\n  /** Reconciliation moved the head, so the observer's totals are restated. */\n  reconciled(info: SQLiteChangeLogInfo): void {\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  state(): SQLiteChangeLogObservabilityState {\n    return {\n      receivedHead: this.#receivedHead,\n      sqliteHead: this.#sqliteHead,\n      headLag: watermarkDistance(this.#receivedHead, this.#sqliteHead),\n      rows: this.#rows,\n      estimatedBytes: this.#estimatedBytes,\n      cookieRows: this.#cookieRows,\n      cookieMutations: this.#cookieMutations,\n      rollbacks: this.#rollbacks,\n      invariantFailures: this.#invariantFailures,\n      hashMatches: this.#hashMatches,\n      hashMismatches: this.#hashMismatches,\n      hashUnpaired: this.#hashUnpaired,\n    };\n  }\n\n  #recordRollback(\n    reason: 'upstream' | 'processing-error' | 'source-interruption',\n  ) {\n    this.#rollbacks++;\n    this.#rollbackCounter.add(1, {reason});\n  }\n\n  #invariantFailure(message: string, details?: Record<string, unknown>) {\n    this.#invariantFailures++;\n    this.#invariantFailureCounter.add(1);\n    this.#lc.error?.(message, details);\n  }\n\n  #compareHashes(\n    watermark: string,\n    sourceHash: string | undefined,\n    sqliteHash: string | undefined,\n  ): void {\n    if (sourceHash === undefined) {\n      this.#recordUnpaired(watermark, 'missing-source-hash');\n      return;\n    }\n    if (sqliteHash === undefined) {\n      this.#recordUnpaired(watermark, 'missing-sqlite-hash');\n      return;\n    }\n    if (sourceHash === sqliteHash) {\n      this.#hashMatches++;\n      this.#hashComparisonCounter.add(1, {outcome: 'match'});\n      return;\n    }\n    this.#hashMismatches++;\n    this.#hashComparisonCounter.add(1, {outcome: 'mismatch'});\n    this.#lc.error?.('SQLite change-log transaction hash mismatch', {\n      sqliteChangeLogHashComparison: {\n        watermark,\n        sourceHash: sourceHash.slice(0, 16),\n        sqliteHash: sqliteHash.slice(0, 16),\n      },\n    });\n  }\n\n  #recordUnpaired(\n    watermark: string,\n    reason:\n      | 'missing-source-hash'\n      | 'missing-sqlite-hash'\n      | 'sqlite-commit-failed',\n  ): void {\n    this.#hashUnpaired++;\n    this.#hashUnpairedCounter.add(1, {reason});\n    this.#lc.warn?.('SQLite change-log transaction hash was not compared', {\n      sqliteChangeLogHashComparison: {watermark, reason},\n    });\n  }\n\n  #resetSourceHash(): void {\n    this.#sourceHasher = undefined;\n    this.#sourcePos = 0;\n    this.#sourceCommitHash = undefined;\n  }\n}\n\nfunction watermarkValue(watermark: string): number | undefined {\n  try {\n    const value = Number(versionFromLexi(watermark));\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n\nfunction watermarkDistance(\n  receivedHead: string,\n  sqliteHead: string,\n): number | undefined {\n  if (receivedHead <= sqliteHead) {\n    return 0;\n  }\n  try {\n    const distance =\n      versionFromLexi(receivedHead) - versionFromLexi(sqliteHead);\n    const value = Number(distance);\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA,SAAgB,+BACd,IACA,QACM;CACN,QAAQ,OAAO,QAAf;EACE,KAAK;GACH,mBACE,WACA,8CACA,kHAEF,EAAE,IAAI,OAAO,IAAI;GACjB;EACF,KAAK;GACH,mBACE,WACA,qCACA,mIAEF,EAAE,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;GAChC;EACF,KAAK,QACH;EACF,SACE,YAAY,MAAM;CACtB;CACA,IAAI,OAAO,cAKT,mBACE,WACA,mCACA,sGAEF,EAAE,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;CAElC,GAAG,QAAQ,oCAAoC,EAC7C,0BAA0B,OAC5B,CAAC;AACH;;;;;;;;AAyBA,SAAgB,8BACd,WAC4B;CAC5B,MAAM,OAAO,kBAAkB,SAAS;CACxC,MAAM,EAAC,cAAc,kBAAiB,oBAAoB,SAAS;CACnE,IAAI,iBAAiB,QAAQ,kBAAkB,MAC7C,MAAM,IAAI,MAAM,yDAAyD;CAE3E,OAAO;EAAC,GAAG;EAAM;EAAc;CAAa;AAC9C;;;;;;;AAQA,SAAgB,uBACd,WACqB;CACrB,MAAM,UAAU,8BAA8B,SAAS;CACvD,MAAM,YAAY,UACf,QAAgB;;;gBAGL,wBAAwB;KACnC,EACA,IAA4C;CAE/C,OAAO;EACL,GAAG;EACH,MAAM,UAAU;EAChB,gBAAgB,UAAU;EAC1B,YAAY,oBAAoB,SAAS;CAC3C;AACF;AAEA,SAAgB,0BACd,IACA,MACA,MACM;CACN,GAAG,OAAO,6BAA6B,EACrC,iBAAiB;EACf;EACA,OAAO,KAAK;EACZ,YAAY,KAAK;EACjB,WAAW,KAAK;EAChB,eAAe,KAAK;EACpB,eAAe,KAAK;EACpB,YAAY,KAAK;EACjB,cAAc,KAAK;EACnB,eAAe,KAAK;CACtB,EACF,CAAC;AACH;;;;;;;;;;AAWA,eAAsB,gBACpB,IACA,MACiB;CAajB,QAAO,MAZa,QAAQ,IAC1B;EAAC;EAAM,GAAG,KAAK;EAAO,GAAG,KAAK;CAAM,EAAE,IAAI,OAAM,SAAQ;EACtD,IAAI;GACF,QAAQ,MAAM,KAAK,IAAI,GAAG;EAC5B,SAAS,GAAG;GACV,IAAK,EAA4B,SAAS,UACxC,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;GAExD,OAAO;EACT;CACF,CAAC,CACH,GACa,QAAQ,OAAO,SAAS,QAAQ,MAAM,CAAC;AACtD;AA4BA,IAAM,0BAA0B;CAAC;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAM;AAAI;;;;;;;AAQ1E,IAAa,0BAAb,MAAqC;CACnC;CACA,qBAA8B,4BAC5B,WACA,iDACA,uEACF;CAMA,qBAA8B,4BAC5B,WACA,yCACA,+KAGF;CACA,mBAA4B,0BAC1B,WACA,sCACA;EACE,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CACF;CACA,mBAA4B,mBAC1B,WACA,+BACA,2DACF;CACA,2BAAoC,mBAClC,WACA,wCACA,uDACF;CACA,yBAAkC,mBAChC,WACA,sCACA,iHACF;CACA,uBAAgC,mBAC9B,WACA,mCACA,2FACF;CAEA,yBAAkC,mBAChC,WACA,sCACA,uEACF;CAEA;CACA;CACA;CACA;CACA;CACA,mBAAmB;CACnB;CACA;CACA,aAAa;CACb;CACA,aAAa;CACb,qBAAqB;CACrB,eAAe;CACf,kBAAkB;CAClB,gBAAgB;CAEhB,YAAY,IAAgB,MAA2B;EACrD,KAAKA,MAAM,GAAG,YAAY,aAAa,4BAA4B;EAGnE,KAAKS,gBAAgB,KAAK;EAC1B,KAAKC,cAAc,KAAK;EACxB,KAAKC,QAAQ,KAAK;EAClB,KAAKC,kBAAkB,KAAK;EAC5B,KAAKC,cAAc,KAAK;EAExB,iBACE,WACA,0BACA,yCACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKF,KAAK,CAAC;EAGlD,iBACE,WACA,iCACA,iEACF,EAAE,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAKE,YAAY,eAAe,EAAC,OAAO,gBAAe,CAAC;GACvE,OAAO,QAAQ,KAAKA,YAAY,aAAa,EAAC,OAAO,cAAa,CAAC;EACrE,CAAC;EACD,iBAAiB,WAAW,oCAAoC;GAC9D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKD,eAAe,CAAC;EAC7D,iBACE,WACA,0BACA,oEACF,EAAE,aAAY,WAAU;GACtB,MAAM,OAAO,eAAe,KAAKF,WAAW;GAC5C,IAAI,SAAS,KAAA,GACX,OAAO,QAAQ,IAAI;EAEvB,CAAC;EACD,iBACE,WACA,8BACA,0JAGF,EAAE,aAAY,WAAU;GACtB,MAAM,MAAM,kBAAkB,KAAKD,eAAe,KAAKC,WAAW;GAClE,IAAI,QAAQ,KAAA,GACV,OAAO,QAAQ,GAAG;EAEtB,CAAC;CACH;;;;;;CAOA,gBACE,MACA,MACA,eAAe,MACf,eAAe,uBAAuB,MAAM,KAAK,GAAG,GAAG,GACjD;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAKI,kBAAkB,KAAA,GACzB,KAAKC,kBACH,iEACA,EAAC,mBAAmB,KAAK,GAAG,gBAAe,CAC7C;GAEF,MAAM,YAAY,KAAK,GAAG;GAC1B,KAAKD,gBAAgB,eACjB,IAAI,2BAA2B,IAC/B,KAAA;GACJ,KAAKE,aAAa;GAClB,KAAKC,oBAAoB,KAAA;GACzB,KAAKH,eAAe,IAAI;IACtB;IACA,KAAK,KAAKE;IACV;IACA,QAAQ;IACR,WAAW;GACb,CAAC;GACD;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAKE,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd,KAAKT,gBAAgB,KAAK,GAAG;EAG/B,IAAI,CAAC,cACH;EAGF,MAAM,YAAY,KAAKU;EACvB,IAAI,KAAKL,kBAAkB,KAAA,KAAa,cAAc,KAAA,GAAW;GAC/D,KAAKC,kBACH,+DACA,EAAC,IAAG,CACN;GACA;EACF;EAEA,MAAM,kBACJ,KAAK,OAAO,WAAW,KAAK,GAAG,YAAY;EAC7C,KAAKD,cAAc,IAAI;GACrB,WAAW;GACX,KAAK,EAAE,KAAKE;GACZ;GACA,QAAQ;GACR,WAAW,KAAK,OAAO,WAAW,YAAY;EAChD,CAAC;EACD,IAAI,KAAK,OAAO,UAAU;GACxB,KAAKC,oBAAoB,KAAKH,cAAc,OAAO;GACnD,KAAKA,gBAAgB,KAAA;EACvB;CACF;;CAGA,iBACE,MACA,QACA,YACM;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,IAAI,eAAe,KAAA,GACjB,KAAKb,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAS,CAAC;EAGxE,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAKkB,0BAA0B,KAAA,GACjC,KAAKJ,kBAAkB,8CAA8C;IACnE,eAAe,KAAKI;IACpB,mBAAmB,KAAK,GAAG;GAC7B,CAAC;GAEH,KAAKA,wBAAwB,KAAK,GAAG;GACrC;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAKC,gBAAgB,UAAU;GAC/B,KAAKD,wBAAwB,KAAA;GAC7B,KAAKD,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd;EAGF,MAAM,YAAY,KAAK,GAAG;EAC1B,IAAI,WAAW,MACb,KAAKhB,mBAAmB,SAAS,OAAO,WAAW;EAErD,IAAI,KAAKiB,0BAA0B,WACjC,KAAKJ,kBACH,8DACA;GAAC,eAAe,KAAKI;GAAuB,iBAAiB;EAAS,CACxE;EAEF,IAAI,WAAW,MAAM;GACnB,KAAKJ,kBACH,gEACA,EAAC,iBAAiB,UAAS,CAC7B;GACA,KAAKM,gBAAgB,WAAW,qBAAqB;EACvD,OAAO;GACL,IAAI,OAAO,cAAc,WACvB,KAAKN,kBACH,+DACA;IAAC,iBAAiB;IAAW,iBAAiB,OAAO;GAAS,CAChE;GAEF,KAAKJ,SAAS,OAAO,MAAM;GAC3B,KAAKC,mBAAmB,OAAO,MAAM;GACrC,KAAKT,iBAAiB,OAAO,OAAO,MAAM,IAAI;GAC9C,IACE,KAAKc,sBAAsB,KAAA,KAC3B,OAAO,MAAM,SAAS,KAAA,GAEtB,KAAKK,eACH,WACA,KAAKL,mBACL,OAAO,MAAM,IACf;EAEJ;EACA,IAAI,YAAY,KAAKP,aACnB,KAAKK,kBAAkB,oCAAoC;GACzD,cAAc,KAAKL;GACnB,iBAAiB;EACnB,CAAC;EAEH,KAAKA,cAAc;EACnB,KAAKS,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;CAEA,cACE,MACA,OACA,YACM;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,IAAI,eAAe,KAAA,GACjB,KAAKjB,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAO,CAAC;EAEtE,IAAI,KAAK,OAAO,YAAY,KAAKgB,sBAAsB,KAAA,GAIrD,KAAKI,gBAAgB,KAAK,GAAG,WAAW,sBAAsB;EAEhE,IACE,iBAAiB,SACjB,MAAM,SAAS,iCAEf,KAAKN,kBAAkB,MAAM,OAAO;EAEtC,IAAI,KAAKI,0BAA0B,KAAA,KAAa,KAAK,OAAO,SAC1D,KAAKC,gBAAgB,kBAAkB;EAEzC,KAAKD,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;;;;;;CAOA,eAAe,KAA6B,MAA6B;EACvE,KAAK,MAAM,MAAM,KAAK;GACpB,KAAKK;GACL,KAAKf,uBAAuB,IAAI,GAAG,EAAC,GAAE,CAAC;EACzC;EACA,KAAKK,cAAc;CACrB;;;;;;;CAQA,iBAAiB,SAAiB,SAAyC;EACzE,KAAKE,kBAAkB,SAAS,OAAO;CACzC;;CAGA,QAAc;EACZ,IAAI,KAAKI,0BAA0B,KAAA,GAAW;GAC5C,KAAKC,gBAAgB,qBAAqB;GAC1C,KAAKD,wBAAwB,KAAA;EAC/B;EACA,KAAKD,iBAAiB;CACxB;;CAGA,WAAW,MAAiC;EAC1C,KAAKT,gBAAgB,KAAK;EAC1B,KAAKC,cAAc,KAAK;EACxB,KAAKC,QAAQ,KAAK;EAClB,KAAKC,kBAAkB,KAAK;EAC5B,KAAKC,cAAc,KAAK;EACxB,KAAKM,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;CAEA,QAA2C;EACzC,OAAO;GACL,cAAc,KAAKT;GACnB,YAAY,KAAKC;GACjB,SAAS,kBAAkB,KAAKD,eAAe,KAAKC,WAAW;GAC/D,MAAM,KAAKC;GACX,gBAAgB,KAAKC;GACrB,YAAY,KAAKC;GACjB,iBAAiB,KAAKU;GACtB,WAAW,KAAKC;GAChB,mBAAmB,KAAKC;GACxB,aAAa,KAAKC;GAClB,gBAAgB,KAAKC;GACrB,cAAc,KAAKC;EACrB;CACF;CAEA,gBACE,QACA;EACA,KAAKJ;EACL,KAAKpB,iBAAiB,IAAI,GAAG,EAAC,OAAM,CAAC;CACvC;CAEA,kBAAkB,SAAiB,SAAmC;EACpE,KAAKqB;EACL,KAAKpB,yBAAyB,IAAI,CAAC;EACnC,KAAKL,IAAI,QAAQ,SAAS,OAAO;CACnC;CAEA,eACE,WACA,YACA,YACM;EACN,IAAI,eAAe,KAAA,GAAW;GAC5B,KAAKqB,gBAAgB,WAAW,qBAAqB;GACrD;EACF;EACA,IAAI,eAAe,KAAA,GAAW;GAC5B,KAAKA,gBAAgB,WAAW,qBAAqB;GACrD;EACF;EACA,IAAI,eAAe,YAAY;GAC7B,KAAKK;GACL,KAAKpB,uBAAuB,IAAI,GAAG,EAAC,SAAS,QAAO,CAAC;GACrD;EACF;EACA,KAAKqB;EACL,KAAKrB,uBAAuB,IAAI,GAAG,EAAC,SAAS,WAAU,CAAC;EACxD,KAAKN,IAAI,QAAQ,+CAA+C,EAC9D,+BAA+B;GAC7B;GACA,YAAY,WAAW,MAAM,GAAG,EAAE;GAClC,YAAY,WAAW,MAAM,GAAG,EAAE;EACpC,EACF,CAAC;CACH;CAEA,gBACE,WACA,QAIM;EACN,KAAK4B;EACL,KAAKrB,qBAAqB,IAAI,GAAG,EAAC,OAAM,CAAC;EACzC,KAAKP,IAAI,OAAO,uDAAuD,EACrE,+BAA+B;GAAC;GAAW;EAAM,EACnD,CAAC;CACH;CAEA,mBAAyB;EACvB,KAAKc,gBAAgB,KAAA;EACrB,KAAKE,aAAa;EAClB,KAAKC,oBAAoB,KAAA;CAC3B;AACF;AAEA,SAAS,eAAe,WAAuC;CAC7D,IAAI;EACF,MAAM,QAAQ,OAAO,gBAAgB,SAAS,CAAC;EAC/C,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF;AAEA,SAAS,kBACP,cACA,YACoB;CACpB,IAAI,gBAAgB,YAClB,OAAO;CAET,IAAI;EACF,MAAM,WACJ,gBAAgB,YAAY,IAAI,gBAAgB,UAAU;EAC5D,MAAM,QAAQ,OAAO,QAAQ;EAC7B,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF"}

package/out/zero-cache/src/services/replicator/write-worker-client.d.ts

@@ -1,6 +1,7 @@
 import type { LogConfig } from '../../../../shared/src/logging.ts';
 import type { Database } from '../../../../zqlite/src/db.ts';
 import type { ChangeStreamData } from '../change-source/protocol/current/downstream.ts';
+import type { BackfillRequest } from '../change-source/protocol/current/upstream.ts';
 import type { ChangeProcessorMode, CommitResult } from './change-processor.ts';
 import type { SubscriptionState } from './schema/replication-state.ts';
 export type PragmaConfig = {
@@ -31,8 +32,15 @@ type ErrorHandler = (err: Error) => void;
 /**
  * Interface for a write worker that processes replication messages.
  */
+/**
+ * The replica's {@link SubscriptionState}, along with its pending backfills,
+ * read in the same snapshot.
+ */
+export type SubscriptionStateWithBackfills = SubscriptionState & {
+    backfills: BackfillRequest[];
+};
 export interface WriteWorkerClient {
-    getSubscriptionState(): Promise<SubscriptionState>;
+    getSubscriptionState(): Promise<SubscriptionStateWithBackfills>;
     processMessages(downstream: readonly ChangeStreamData[]): Promise<CommitResult | null>;
     abort(): void;
     stop(): Promise<void>;
@@ -67,7 +75,7 @@ export type Request<M extends Method = Method> = {
 };
 export type ResultMap = {
     init: void;
-    getSubscriptionState: SubscriptionState;
+    getSubscriptionState: SubscriptionStateWithBackfills;
     processMessages: CommitResult | null;
     abort: void;
     stop: void;
@@ -93,7 +101,7 @@ export declare class ThreadWriteWorkerClient implements WriteWorkerClient {
     #private;
     constructor();
     init(dbPath: string, mode: ChangeProcessorMode, pragmas: PragmaConfig, logConfig: LogConfig, checkpoint?: ForceCheckpointConfig | null): Promise<void>;
-    getSubscriptionState(): Promise<SubscriptionState>;
+    getSubscriptionState(): Promise<SubscriptionStateWithBackfills>;
     processMessages(downstream: readonly ChangeStreamData[]): Promise<CommitResult | null>;
     abort(): void;
     stop(): Promise<void>;

package/out/zero-cache/src/services/replicator/write-worker-client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"write-worker-client.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAE3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,mBAAmB,EAAE,YAAY,EAAC,MAAM,uBAAuB,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAErE,MAAM,MAAM,YAAY,GAAG;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC,CAAC;AAEF;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,qBAAqB,GAAG;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,KAAK,YAAY,GAAG,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,oBAAoB,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACnD,eAAe,CACb,UAAU,EAAE,SAAS,gBAAgB,EAAE,GACtC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChC,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,KAAK,CAAC,EAAE,eAAe,GAAG,MAAM,GAAG,SAAS,CAAC;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CAC/C,CAAC;AAEF,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,eAAe,CA8B5D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,eAAe,GAAG,KAAK,CAgBnE;AAGD,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE;QACJ,MAAM;QACN,mBAAmB;QACnB,YAAY;QACZ,SAAS;QACT,qBAAqB,GAAG,IAAI;KAC7B,CAAC;IACF,oBAAoB,EAAE,EAAE,CAAC;IACzB,eAAe,EAAE,CAAC,SAAS,gBAAgB,EAAE,CAAC,CAAC;IAC/C,KAAK,EAAE,EAAE,CAAC;IACV,IAAI,EAAE,EAAE,CAAC;CACV,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;AAEnC,MAAM,MAAM,OAAO,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;CAAC,CAAC;AAE/E,MAAM,MAAM,SAAS,GAAG;IACtB,IAAI,EAAE,IAAI,CAAC;IACX,oBAAoB,EAAE,iBAAiB,CAAC;IACxC,eAAe,EAAE,YAAY,GAAG,IAAI,CAAC;IACrC,KAAK,EAAE,IAAI,CAAC;IACZ,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,QAAQ,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAC1C;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,SAAS,CAAA;CAAC,GACpD;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,KAAK,EAAE,eAAe,CAAC;IAAC,MAAM,CAAC,EAAE,SAAS,CAAA;CAAC,CAAC;AAE5D,MAAM,MAAM,UAAU,GAAG;IAAC,UAAU,EAAE,eAAe,CAAA;CAAC,CAAC;AAEvD,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,QAM/D;AAED;;;GAGG;AACH,qBAAa,uBAAwB,YAAW,iBAAiB;;IAM/D,cAiCC;IAkBD,IAAI,CACF,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,mBAAmB,EACzB,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,SAAS,EACpB,UAAU,GAAE,qBAAqB,GAAG,IAAW,GAC9C,OAAO,CAAC,IAAI,CAAC,CAEf;IAED,oBAAoB,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAEjD;IAED,eAAe,CACb,UAAU,EAAE,SAAS,gBAAgB,EAAE,GACtC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAE9B;IAED,KAAK,IAAI,IAAI,CAIZ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAK1B;IAED,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAEnC;CACF"}
+{"version":3,"file":"write-worker-client.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAE3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AACnF,OAAO,KAAK,EAAC,mBAAmB,EAAE,YAAY,EAAC,MAAM,uBAAuB,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAErE,MAAM,MAAM,YAAY,GAAG;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC,CAAC;AAEF;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,qBAAqB,GAAG;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,KAAK,YAAY,GAAG,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,CAAC;AAEzC;;GAEG;AACH;;;GAGG;AACH,MAAM,MAAM,8BAA8B,GAAG,iBAAiB,GAAG;IAC/D,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B,CAAC;AAEF,MAAM,WAAW,iBAAiB;IAChC,oBAAoB,IAAI,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAChE,eAAe,CACb,UAAU,EAAE,SAAS,gBAAgB,EAAE,GACtC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChC,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,KAAK,CAAC,EAAE,eAAe,GAAG,MAAM,GAAG,SAAS,CAAC;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CAC/C,CAAC;AAEF,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,eAAe,CA8B5D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,eAAe,GAAG,KAAK,CAgBnE;AAGD,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE;QACJ,MAAM;QACN,mBAAmB;QACnB,YAAY;QACZ,SAAS;QACT,qBAAqB,GAAG,IAAI;KAC7B,CAAC;IACF,oBAAoB,EAAE,EAAE,CAAC;IACzB,eAAe,EAAE,CAAC,SAAS,gBAAgB,EAAE,CAAC,CAAC;IAC/C,KAAK,EAAE,EAAE,CAAC;IACV,IAAI,EAAE,EAAE,CAAC;CACV,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;AAEnC,MAAM,MAAM,OAAO,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;CAAC,CAAC;AAE/E,MAAM,MAAM,SAAS,GAAG;IACtB,IAAI,EAAE,IAAI,CAAC;IACX,oBAAoB,EAAE,8BAA8B,CAAC;IACrD,eAAe,EAAE,YAAY,GAAG,IAAI,CAAC;IACrC,KAAK,EAAE,IAAI,CAAC;IACZ,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,QAAQ,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,IAC1C;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,SAAS,CAAA;CAAC,GACpD;IAAC,MAAM,EAAE,CAAC,CAAC;IAAC,KAAK,EAAE,eAAe,CAAC;IAAC,MAAM,CAAC,EAAE,SAAS,CAAA;CAAC,CAAC;AAE5D,MAAM,MAAM,UAAU,GAAG;IAAC,UAAU,EAAE,eAAe,CAAA;CAAC,CAAC;AAEvD,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,QAM/D;AAED;;;GAGG;AACH,qBAAa,uBAAwB,YAAW,iBAAiB;;IAM/D,cAiCC;IAkBD,IAAI,CACF,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,mBAAmB,EACzB,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,SAAS,EACpB,UAAU,GAAE,qBAAqB,GAAG,IAAW,GAC9C,OAAO,CAAC,IAAI,CAAC,CAEf;IAED,oBAAoB,IAAI,OAAO,CAAC,8BAA8B,CAAC,CAE9D;IAED,eAAe,CACb,UAAU,EAAE,SAAS,gBAAgB,EAAE,GACtC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAE9B;IAED,KAAK,IAAI,IAAI,CAIZ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAK1B;IAED,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAEnC;CACF"}

package/out/zero-cache/src/services/replicator/write-worker-client.js.map

@@ -1 +1 @@
-{"version":3,"file":"write-worker-client.js","names":["#worker","#rejectAll","#errorHandler","#pending","#terminated","#call"],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"sourcesContent":["import {Worker} from 'node:worker_threads';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {WRITE_WORKER_URL} from '../../server/worker-urls.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeProcessorMode, CommitResult} from './change-processor.ts';\nimport type {SubscriptionState} from './schema/replication-state.ts';\n\nexport type PragmaConfig = {\n  busyTimeout: number;\n  analysisLimit: number;\n  walAutocheckpoint?: number | undefined;\n};\n\n/**\n * Enables write-path checkpoint backpressure in the write worker. At each\n * commit boundary, if the WAL has grown to at least `checkpointThresholdPages`,\n * the worker forces an immediate litestream sync (sealing the WAL to LTX and\n * running litestream's checkpoint).\n *\n * The optional `walMaxPages` is an emergency break that pauses until the WAL drains,\n * rather than letting it exceed available disk space. This replaces litestream's\n * analogous `truncate-page-n` emergency break, which is more disruptive because of\n * https://github.com/benbjohnson/litestream/issues/1332. Note, however, this should\n * never been needed if non-initial snapshots are disabled (which is the default).\n *\n * Set only for the backup replicator when replicating with litestream v5; `null`\n * disables the feature entirely (no per-commit overhead).\n */\nexport type ForceCheckpointConfig = {\n  checkpointThresholdPages: number;\n  maxWalPages?: number | undefined;\n};\n\ntype ErrorHandler = (err: Error) => void;\n\n/**\n * Interface for a write worker that processes replication messages.\n */\nexport interface WriteWorkerClient {\n  getSubscriptionState(): Promise<SubscriptionState>;\n  processMessages(\n    downstream: readonly ChangeStreamData[],\n  ): Promise<CommitResult | null>;\n  abort(): void;\n  stop(): Promise<void>;\n  onError(handler: ErrorHandler): void;\n}\n\nexport type SerializedError = {\n  name: string;\n  message: string;\n  stack?: string | undefined;\n  cause?: SerializedError | string | undefined;\n  details?: Record<string, unknown> | undefined;\n};\n\nexport function serializeError(err: unknown): SerializedError {\n  if (!(err instanceof Error)) {\n    return {\n      name: 'Error',\n      message: String(err),\n      details: err && typeof err === 'object' ? {...err} : undefined,\n    };\n  }\n\n  // Error fields such as message, stack, and some native error details are\n  // non-enumerable, so JSON.stringify(err) would usually return \"{}\".\n  const details = Object.fromEntries(\n    Object.getOwnPropertyNames(err)\n      .filter(key => !['name', 'message', 'stack', 'cause'].includes(key))\n      .map(key => [key, (err as unknown as Record<string, unknown>)[key]]),\n  );\n  const cause =\n    err.cause instanceof Error\n      ? serializeError(err.cause)\n      : err.cause === undefined\n        ? undefined\n        : String(err.cause);\n\n  return {\n    name: err.name,\n    message: err.message,\n    stack: err.stack,\n    cause,\n    details: Object.keys(details).length ? details : undefined,\n  };\n}\n\nexport function deserializeError(serialized: SerializedError): Error {\n  const err = new Error(serialized.message);\n  err.name = serialized.name;\n  if (serialized.stack !== undefined) {\n    err.stack = serialized.stack;\n  }\n  if (serialized.cause !== undefined) {\n    err.cause =\n      typeof serialized.cause === 'string'\n        ? serialized.cause\n        : deserializeError(serialized.cause);\n  }\n  if (serialized.details) {\n    Object.assign(err, serialized.details);\n  }\n  return err;\n}\n\n// Wire protocol types.\nexport type ArgsMap = {\n  init: [\n    string,\n    ChangeProcessorMode,\n    PragmaConfig,\n    LogConfig,\n    ForceCheckpointConfig | null,\n  ];\n  getSubscriptionState: [];\n  processMessages: [readonly ChangeStreamData[]];\n  abort: [];\n  stop: [];\n};\n\nexport type Method = keyof ArgsMap;\n\nexport type Request<M extends Method = Method> = {method: M; args: ArgsMap[M]};\n\nexport type ResultMap = {\n  init: void;\n  getSubscriptionState: SubscriptionState;\n  processMessages: CommitResult | null;\n  abort: void;\n  stop: void;\n};\n\nexport type Response<M extends Method = Method> =\n  | {method: M; result: ResultMap[M]; error?: undefined}\n  | {method: M; error: SerializedError; result?: undefined};\n\nexport type WriteError = {writeError: SerializedError};\n\nexport function applyPragmas(db: Database, pragmas: PragmaConfig) {\n  db.pragma(`busy_timeout = ${pragmas.busyTimeout}`);\n  db.pragma(`analysis_limit = ${pragmas.analysisLimit}`);\n  if (pragmas.walAutocheckpoint !== undefined) {\n    db.pragma(`wal_autocheckpoint = ${pragmas.walAutocheckpoint}`);\n  }\n}\n\n/**\n * Delegates SQLite writes to a worker_thread,\n * keeping the main event loop free for WebSocket heartbeats and IPC.\n */\nexport class ThreadWriteWorkerClient implements WriteWorkerClient {\n  readonly #worker: Worker;\n  #pending: Resolver<unknown, Error> | null = null;\n  #errorHandler: ErrorHandler = () => {};\n  #terminated = false;\n\n  constructor() {\n    this.#worker = new Worker(WRITE_WORKER_URL);\n\n    this.#worker.on('message', (msg: Response | WriteError) => {\n      if ('writeError' in msg) {\n        const error = deserializeError(msg.writeError);\n        this.#rejectAll(error);\n        this.#errorHandler(error);\n        return;\n      }\n      const r = this.#pending;\n      if (!r) return; // stale abort response\n      this.#pending = null;\n      if (msg.error !== undefined) {\n        r.reject(deserializeError(msg.error));\n      } else {\n        r.resolve(msg.result);\n      }\n    });\n\n    this.#worker.on('error', (err: Error) => {\n      this.#rejectAll(err);\n      this.#errorHandler(err);\n    });\n\n    this.#worker.on('exit', (code: number) => {\n      this.#terminated = true;\n      if (code !== 0) {\n        const err = new Error(`Worker exited with code ${code}`);\n        this.#rejectAll(err);\n        this.#errorHandler(err);\n      }\n    });\n  }\n\n  #rejectAll(err: Error) {\n    const r = this.#pending;\n    if (r) {\n      this.#pending = null;\n      r.reject(err);\n    }\n  }\n\n  #call<M extends Method>(method: M, args: ArgsMap[M]): Promise<ResultMap[M]> {\n    assert(this.#pending === null, `concurrent call: ${method}`);\n    const r = resolver<ResultMap[M]>();\n    this.#pending = r as Resolver<unknown, Error>;\n    this.#worker.postMessage({method, args} satisfies Request);\n    return r.promise;\n  }\n\n  init(\n    dbPath: string,\n    mode: ChangeProcessorMode,\n    pragmas: PragmaConfig,\n    logConfig: LogConfig,\n    checkpoint: ForceCheckpointConfig | null = null,\n  ): Promise<void> {\n    return this.#call('init', [dbPath, mode, pragmas, logConfig, checkpoint]);\n  }\n\n  getSubscriptionState(): Promise<SubscriptionState> {\n    return this.#call('getSubscriptionState', []);\n  }\n\n  processMessages(\n    downstream: readonly ChangeStreamData[],\n  ): Promise<CommitResult | null> {\n    return this.#call('processMessages', [downstream]);\n  }\n\n  abort(): void {\n    if (!this.#terminated) {\n      this.#worker.postMessage({method: 'abort', args: []} satisfies Request);\n    }\n  }\n\n  async stop(): Promise<void> {\n    await this.#call('stop', []);\n    if (!this.#terminated) {\n      await this.#worker.terminate();\n    }\n  }\n\n  onError(handler: ErrorHandler): void {\n    this.#errorHandler = handler;\n  }\n}\n"],"mappings":";;;;;AA2DA,SAAgB,eAAe,KAA+B;CAC5D,IAAI,EAAE,eAAe,QACnB,OAAO;EACL,MAAM;EACN,SAAS,OAAO,GAAG;EACnB,SAAS,OAAO,OAAO,QAAQ,WAAW,EAAC,GAAG,IAAG,IAAI,KAAA;CACvD;CAKF,MAAM,UAAU,OAAO,YACrB,OAAO,oBAAoB,GAAG,EAC3B,QAAO,QAAO,CAAC;EAAC;EAAQ;EAAW;EAAS;CAAO,EAAE,SAAS,GAAG,CAAC,EAClE,KAAI,QAAO,CAAC,KAAM,IAA2C,IAAI,CAAC,CACvE;CACA,MAAM,QACJ,IAAI,iBAAiB,QACjB,eAAe,IAAI,KAAK,IACxB,IAAI,UAAU,KAAA,IACZ,KAAA,IACA,OAAO,IAAI,KAAK;CAExB,OAAO;EACL,MAAM,IAAI;EACV,SAAS,IAAI;EACb,OAAO,IAAI;EACX;EACA,SAAS,OAAO,KAAK,OAAO,EAAE,SAAS,UAAU,KAAA;CACnD;AACF;AAEA,SAAgB,iBAAiB,YAAoC;CACnE,MAAM,MAAM,IAAI,MAAM,WAAW,OAAO;CACxC,IAAI,OAAO,WAAW;CACtB,IAAI,WAAW,UAAU,KAAA,GACvB,IAAI,QAAQ,WAAW;CAEzB,IAAI,WAAW,UAAU,KAAA,GACvB,IAAI,QACF,OAAO,WAAW,UAAU,WACxB,WAAW,QACX,iBAAiB,WAAW,KAAK;CAEzC,IAAI,WAAW,SACb,OAAO,OAAO,KAAK,WAAW,OAAO;CAEvC,OAAO;AACT;AAmCA,SAAgB,aAAa,IAAc,SAAuB;CAChE,GAAG,OAAO,kBAAkB,QAAQ,aAAa;CACjD,GAAG,OAAO,oBAAoB,QAAQ,eAAe;CACrD,IAAI,QAAQ,sBAAsB,KAAA,GAChC,GAAG,OAAO,wBAAwB,QAAQ,mBAAmB;AAEjE;;;;;AAMA,IAAa,0BAAb,MAAkE;CAChE;CACA,WAA4C;CAC5C,sBAAoC,CAAC;CACrC,cAAc;CAEd,cAAc;EACZ,KAAKA,UAAU,IAAI,OAAO,gBAAgB;EAE1C,KAAKA,QAAQ,GAAG,YAAY,QAA+B;GACzD,IAAI,gBAAgB,KAAK;IACvB,MAAM,QAAQ,iBAAiB,IAAI,UAAU;IAC7C,KAAKC,WAAW,KAAK;IACrB,KAAKC,cAAc,KAAK;IACxB;GACF;GACA,MAAM,IAAI,KAAKC;GACf,IAAI,CAAC,GAAG;GACR,KAAKA,WAAW;GAChB,IAAI,IAAI,UAAU,KAAA,GAChB,EAAE,OAAO,iBAAiB,IAAI,KAAK,CAAC;QAEpC,EAAE,QAAQ,IAAI,MAAM;EAExB,CAAC;EAED,KAAKH,QAAQ,GAAG,UAAU,QAAe;GACvC,KAAKC,WAAW,GAAG;GACnB,KAAKC,cAAc,GAAG;EACxB,CAAC;EAED,KAAKF,QAAQ,GAAG,SAAS,SAAiB;GACxC,KAAKI,cAAc;GACnB,IAAI,SAAS,GAAG;IACd,MAAM,sBAAM,IAAI,MAAM,2BAA2B,MAAM;IACvD,KAAKH,WAAW,GAAG;IACnB,KAAKC,cAAc,GAAG;GACxB;EACF,CAAC;CACH;CAEA,WAAW,KAAY;EACrB,MAAM,IAAI,KAAKC;EACf,IAAI,GAAG;GACL,KAAKA,WAAW;GAChB,EAAE,OAAO,GAAG;EACd;CACF;CAEA,MAAwB,QAAW,MAAyC;EAC1E,OAAO,KAAKA,aAAa,MAAM,oBAAoB,QAAQ;EAC3D,MAAM,IAAI,SAAuB;EACjC,KAAKA,WAAW;EAChB,KAAKH,QAAQ,YAAY;GAAC;GAAQ;EAAI,CAAmB;EACzD,OAAO,EAAE;CACX;CAEA,KACE,QACA,MACA,SACA,WACA,aAA2C,MAC5B;EACf,OAAO,KAAKK,MAAM,QAAQ;GAAC;GAAQ;GAAM;GAAS;GAAW;EAAU,CAAC;CAC1E;CAEA,uBAAmD;EACjD,OAAO,KAAKA,MAAM,wBAAwB,CAAC,CAAC;CAC9C;CAEA,gBACE,YAC8B;EAC9B,OAAO,KAAKA,MAAM,mBAAmB,CAAC,UAAU,CAAC;CACnD;CAEA,QAAc;EACZ,IAAI,CAAC,KAAKD,aACR,KAAKJ,QAAQ,YAAY;GAAC,QAAQ;GAAS,MAAM,CAAC;EAAC,CAAmB;CAE1E;CAEA,MAAM,OAAsB;EAC1B,MAAM,KAAKK,MAAM,QAAQ,CAAC,CAAC;EAC3B,IAAI,CAAC,KAAKD,aACR,MAAM,KAAKJ,QAAQ,UAAU;CAEjC;CAEA,QAAQ,SAA6B;EACnC,KAAKE,gBAAgB;CACvB;AACF"}
+{"version":3,"file":"write-worker-client.js","names":["#worker","#rejectAll","#errorHandler","#pending","#terminated","#call"],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker-client.ts"],"sourcesContent":["import {Worker} from 'node:worker_threads';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {WRITE_WORKER_URL} from '../../server/worker-urls.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport type {ChangeProcessorMode, CommitResult} from './change-processor.ts';\nimport type {SubscriptionState} from './schema/replication-state.ts';\n\nexport type PragmaConfig = {\n  busyTimeout: number;\n  analysisLimit: number;\n  walAutocheckpoint?: number | undefined;\n};\n\n/**\n * Enables write-path checkpoint backpressure in the write worker. At each\n * commit boundary, if the WAL has grown to at least `checkpointThresholdPages`,\n * the worker forces an immediate litestream sync (sealing the WAL to LTX and\n * running litestream's checkpoint).\n *\n * The optional `walMaxPages` is an emergency break that pauses until the WAL drains,\n * rather than letting it exceed available disk space. This replaces litestream's\n * analogous `truncate-page-n` emergency break, which is more disruptive because of\n * https://github.com/benbjohnson/litestream/issues/1332. Note, however, this should\n * never been needed if non-initial snapshots are disabled (which is the default).\n *\n * Set only for the backup replicator when replicating with litestream v5; `null`\n * disables the feature entirely (no per-commit overhead).\n */\nexport type ForceCheckpointConfig = {\n  checkpointThresholdPages: number;\n  maxWalPages?: number | undefined;\n};\n\ntype ErrorHandler = (err: Error) => void;\n\n/**\n * Interface for a write worker that processes replication messages.\n */\n/**\n * The replica's {@link SubscriptionState}, along with its pending backfills,\n * read in the same snapshot.\n */\nexport type SubscriptionStateWithBackfills = SubscriptionState & {\n  backfills: BackfillRequest[];\n};\n\nexport interface WriteWorkerClient {\n  getSubscriptionState(): Promise<SubscriptionStateWithBackfills>;\n  processMessages(\n    downstream: readonly ChangeStreamData[],\n  ): Promise<CommitResult | null>;\n  abort(): void;\n  stop(): Promise<void>;\n  onError(handler: ErrorHandler): void;\n}\n\nexport type SerializedError = {\n  name: string;\n  message: string;\n  stack?: string | undefined;\n  cause?: SerializedError | string | undefined;\n  details?: Record<string, unknown> | undefined;\n};\n\nexport function serializeError(err: unknown): SerializedError {\n  if (!(err instanceof Error)) {\n    return {\n      name: 'Error',\n      message: String(err),\n      details: err && typeof err === 'object' ? {...err} : undefined,\n    };\n  }\n\n  // Error fields such as message, stack, and some native error details are\n  // non-enumerable, so JSON.stringify(err) would usually return \"{}\".\n  const details = Object.fromEntries(\n    Object.getOwnPropertyNames(err)\n      .filter(key => !['name', 'message', 'stack', 'cause'].includes(key))\n      .map(key => [key, (err as unknown as Record<string, unknown>)[key]]),\n  );\n  const cause =\n    err.cause instanceof Error\n      ? serializeError(err.cause)\n      : err.cause === undefined\n        ? undefined\n        : String(err.cause);\n\n  return {\n    name: err.name,\n    message: err.message,\n    stack: err.stack,\n    cause,\n    details: Object.keys(details).length ? details : undefined,\n  };\n}\n\nexport function deserializeError(serialized: SerializedError): Error {\n  const err = new Error(serialized.message);\n  err.name = serialized.name;\n  if (serialized.stack !== undefined) {\n    err.stack = serialized.stack;\n  }\n  if (serialized.cause !== undefined) {\n    err.cause =\n      typeof serialized.cause === 'string'\n        ? serialized.cause\n        : deserializeError(serialized.cause);\n  }\n  if (serialized.details) {\n    Object.assign(err, serialized.details);\n  }\n  return err;\n}\n\n// Wire protocol types.\nexport type ArgsMap = {\n  init: [\n    string,\n    ChangeProcessorMode,\n    PragmaConfig,\n    LogConfig,\n    ForceCheckpointConfig | null,\n  ];\n  getSubscriptionState: [];\n  processMessages: [readonly ChangeStreamData[]];\n  abort: [];\n  stop: [];\n};\n\nexport type Method = keyof ArgsMap;\n\nexport type Request<M extends Method = Method> = {method: M; args: ArgsMap[M]};\n\nexport type ResultMap = {\n  init: void;\n  getSubscriptionState: SubscriptionStateWithBackfills;\n  processMessages: CommitResult | null;\n  abort: void;\n  stop: void;\n};\n\nexport type Response<M extends Method = Method> =\n  | {method: M; result: ResultMap[M]; error?: undefined}\n  | {method: M; error: SerializedError; result?: undefined};\n\nexport type WriteError = {writeError: SerializedError};\n\nexport function applyPragmas(db: Database, pragmas: PragmaConfig) {\n  db.pragma(`busy_timeout = ${pragmas.busyTimeout}`);\n  db.pragma(`analysis_limit = ${pragmas.analysisLimit}`);\n  if (pragmas.walAutocheckpoint !== undefined) {\n    db.pragma(`wal_autocheckpoint = ${pragmas.walAutocheckpoint}`);\n  }\n}\n\n/**\n * Delegates SQLite writes to a worker_thread,\n * keeping the main event loop free for WebSocket heartbeats and IPC.\n */\nexport class ThreadWriteWorkerClient implements WriteWorkerClient {\n  readonly #worker: Worker;\n  #pending: Resolver<unknown, Error> | null = null;\n  #errorHandler: ErrorHandler = () => {};\n  #terminated = false;\n\n  constructor() {\n    this.#worker = new Worker(WRITE_WORKER_URL);\n\n    this.#worker.on('message', (msg: Response | WriteError) => {\n      if ('writeError' in msg) {\n        const error = deserializeError(msg.writeError);\n        this.#rejectAll(error);\n        this.#errorHandler(error);\n        return;\n      }\n      const r = this.#pending;\n      if (!r) return; // stale abort response\n      this.#pending = null;\n      if (msg.error !== undefined) {\n        r.reject(deserializeError(msg.error));\n      } else {\n        r.resolve(msg.result);\n      }\n    });\n\n    this.#worker.on('error', (err: Error) => {\n      this.#rejectAll(err);\n      this.#errorHandler(err);\n    });\n\n    this.#worker.on('exit', (code: number) => {\n      this.#terminated = true;\n      if (code !== 0) {\n        const err = new Error(`Worker exited with code ${code}`);\n        this.#rejectAll(err);\n        this.#errorHandler(err);\n      }\n    });\n  }\n\n  #rejectAll(err: Error) {\n    const r = this.#pending;\n    if (r) {\n      this.#pending = null;\n      r.reject(err);\n    }\n  }\n\n  #call<M extends Method>(method: M, args: ArgsMap[M]): Promise<ResultMap[M]> {\n    assert(this.#pending === null, `concurrent call: ${method}`);\n    const r = resolver<ResultMap[M]>();\n    this.#pending = r as Resolver<unknown, Error>;\n    this.#worker.postMessage({method, args} satisfies Request);\n    return r.promise;\n  }\n\n  init(\n    dbPath: string,\n    mode: ChangeProcessorMode,\n    pragmas: PragmaConfig,\n    logConfig: LogConfig,\n    checkpoint: ForceCheckpointConfig | null = null,\n  ): Promise<void> {\n    return this.#call('init', [dbPath, mode, pragmas, logConfig, checkpoint]);\n  }\n\n  getSubscriptionState(): Promise<SubscriptionStateWithBackfills> {\n    return this.#call('getSubscriptionState', []);\n  }\n\n  processMessages(\n    downstream: readonly ChangeStreamData[],\n  ): Promise<CommitResult | null> {\n    return this.#call('processMessages', [downstream]);\n  }\n\n  abort(): void {\n    if (!this.#terminated) {\n      this.#worker.postMessage({method: 'abort', args: []} satisfies Request);\n    }\n  }\n\n  async stop(): Promise<void> {\n    await this.#call('stop', []);\n    if (!this.#terminated) {\n      await this.#worker.terminate();\n    }\n  }\n\n  onError(handler: ErrorHandler): void {\n    this.#errorHandler = handler;\n  }\n}\n"],"mappings":";;;;;AAoEA,SAAgB,eAAe,KAA+B;CAC5D,IAAI,EAAE,eAAe,QACnB,OAAO;EACL,MAAM;EACN,SAAS,OAAO,GAAG;EACnB,SAAS,OAAO,OAAO,QAAQ,WAAW,EAAC,GAAG,IAAG,IAAI,KAAA;CACvD;CAKF,MAAM,UAAU,OAAO,YACrB,OAAO,oBAAoB,GAAG,EAC3B,QAAO,QAAO,CAAC;EAAC;EAAQ;EAAW;EAAS;CAAO,EAAE,SAAS,GAAG,CAAC,EAClE,KAAI,QAAO,CAAC,KAAM,IAA2C,IAAI,CAAC,CACvE;CACA,MAAM,QACJ,IAAI,iBAAiB,QACjB,eAAe,IAAI,KAAK,IACxB,IAAI,UAAU,KAAA,IACZ,KAAA,IACA,OAAO,IAAI,KAAK;CAExB,OAAO;EACL,MAAM,IAAI;EACV,SAAS,IAAI;EACb,OAAO,IAAI;EACX;EACA,SAAS,OAAO,KAAK,OAAO,EAAE,SAAS,UAAU,KAAA;CACnD;AACF;AAEA,SAAgB,iBAAiB,YAAoC;CACnE,MAAM,MAAM,IAAI,MAAM,WAAW,OAAO;CACxC,IAAI,OAAO,WAAW;CACtB,IAAI,WAAW,UAAU,KAAA,GACvB,IAAI,QAAQ,WAAW;CAEzB,IAAI,WAAW,UAAU,KAAA,GACvB,IAAI,QACF,OAAO,WAAW,UAAU,WACxB,WAAW,QACX,iBAAiB,WAAW,KAAK;CAEzC,IAAI,WAAW,SACb,OAAO,OAAO,KAAK,WAAW,OAAO;CAEvC,OAAO;AACT;AAmCA,SAAgB,aAAa,IAAc,SAAuB;CAChE,GAAG,OAAO,kBAAkB,QAAQ,aAAa;CACjD,GAAG,OAAO,oBAAoB,QAAQ,eAAe;CACrD,IAAI,QAAQ,sBAAsB,KAAA,GAChC,GAAG,OAAO,wBAAwB,QAAQ,mBAAmB;AAEjE;;;;;AAMA,IAAa,0BAAb,MAAkE;CAChE;CACA,WAA4C;CAC5C,sBAAoC,CAAC;CACrC,cAAc;CAEd,cAAc;EACZ,KAAKA,UAAU,IAAI,OAAO,gBAAgB;EAE1C,KAAKA,QAAQ,GAAG,YAAY,QAA+B;GACzD,IAAI,gBAAgB,KAAK;IACvB,MAAM,QAAQ,iBAAiB,IAAI,UAAU;IAC7C,KAAKC,WAAW,KAAK;IACrB,KAAKC,cAAc,KAAK;IACxB;GACF;GACA,MAAM,IAAI,KAAKC;GACf,IAAI,CAAC,GAAG;GACR,KAAKA,WAAW;GAChB,IAAI,IAAI,UAAU,KAAA,GAChB,EAAE,OAAO,iBAAiB,IAAI,KAAK,CAAC;QAEpC,EAAE,QAAQ,IAAI,MAAM;EAExB,CAAC;EAED,KAAKH,QAAQ,GAAG,UAAU,QAAe;GACvC,KAAKC,WAAW,GAAG;GACnB,KAAKC,cAAc,GAAG;EACxB,CAAC;EAED,KAAKF,QAAQ,GAAG,SAAS,SAAiB;GACxC,KAAKI,cAAc;GACnB,IAAI,SAAS,GAAG;IACd,MAAM,sBAAM,IAAI,MAAM,2BAA2B,MAAM;IACvD,KAAKH,WAAW,GAAG;IACnB,KAAKC,cAAc,GAAG;GACxB;EACF,CAAC;CACH;CAEA,WAAW,KAAY;EACrB,MAAM,IAAI,KAAKC;EACf,IAAI,GAAG;GACL,KAAKA,WAAW;GAChB,EAAE,OAAO,GAAG;EACd;CACF;CAEA,MAAwB,QAAW,MAAyC;EAC1E,OAAO,KAAKA,aAAa,MAAM,oBAAoB,QAAQ;EAC3D,MAAM,IAAI,SAAuB;EACjC,KAAKA,WAAW;EAChB,KAAKH,QAAQ,YAAY;GAAC;GAAQ;EAAI,CAAmB;EACzD,OAAO,EAAE;CACX;CAEA,KACE,QACA,MACA,SACA,WACA,aAA2C,MAC5B;EACf,OAAO,KAAKK,MAAM,QAAQ;GAAC;GAAQ;GAAM;GAAS;GAAW;EAAU,CAAC;CAC1E;CAEA,uBAAgE;EAC9D,OAAO,KAAKA,MAAM,wBAAwB,CAAC,CAAC;CAC9C;CAEA,gBACE,YAC8B;EAC9B,OAAO,KAAKA,MAAM,mBAAmB,CAAC,UAAU,CAAC;CACnD;CAEA,QAAc;EACZ,IAAI,CAAC,KAAKD,aACR,KAAKJ,QAAQ,YAAY;GAAC,QAAQ;GAAS,MAAM,CAAC;EAAC,CAAmB;CAE1E;CAEA,MAAM,OAAsB;EAC1B,MAAM,KAAKK,MAAM,QAAQ,CAAC,CAAC;EAC3B,IAAI,CAAC,KAAKD,aACR,MAAM,KAAKJ,QAAQ,UAAU;CAEjC;CAEA,QAAQ,SAA6B;EACnC,KAAKE,gBAAgB;CACvB;AACF"}