@rocicorp/zero 1.11.0-canary.0 → 1.11.0-canary.10

package/out/zero-cache/src/services/replicator/replication-status.d.ts

@@ -1,7 +1,8 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { JSONObject } from '../../../../zero-events/src/json.ts';
-import type { ReplicationStage, ReplicationState, ReplicationStatusEvent, Status } from '../../../../zero-events/src/status.ts';
+import type { IndexingStatus, ReplicationStage, ReplicationState, ReplicationStatusEvent, Status } from '../../../../zero-events/src/status.ts';
 import { Database } from '../../../../zqlite/src/db.ts';
+import type { LiteIndexSpec } from '../../db/specs.ts';
 import { publishCriticalEvent } from '../../observability/events.ts';
 export declare class ReplicationStatusPublisher {
     #private;
@@ -10,9 +11,42 @@ export declare class ReplicationStatusPublisher {
     static forReplicaFile(file: string, publishFn?: typeof publishCriticalEvent): ReplicationStatusPublisher;
     constructor(dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T, publishFn?: typeof publishCriticalEvent);
     publish(lc: LogContext, stage: ReplicationStage, description?: string, interval?: number, extraState?: () => Partial<ReplicationState>, now?: Date): this;
+    /**
+     * Like {@link publish}, but waits (for at most `maxWaitMs`) for the event
+     * to be sent. Use this before a long synchronous operation (e.g. creating
+     * an index) that blocks the event loop and would otherwise prevent the
+     * event from being sent until the operation completes.
+     */
+    publishAndFlush(lc: LogContext, stage: ReplicationStage, description?: string, interval?: number, extraState?: () => Partial<ReplicationState>, maxWaitMs?: number): Promise<void>;
     publishAndThrowError(lc: LogContext, stage: ReplicationStage, e: unknown): Promise<never>;
     stop(): this;
 }
+/**
+ * Tracks the progress of a sequence of index creations for reporting in
+ * the `indexingStatus` field of {@link ReplicationState}.
+ */
+export declare class IndexingProgress {
+    #private;
+    /**
+     * @param totalIndexes The total number of indexes to be created, if known.
+     */
+    constructor(totalIndexes?: number, now?: () => number);
+    /** Marks the start of creating the `index`. */
+    start(index: LiteIndexSpec): void;
+    /**
+     * Restarts the timer of the current index, e.g. to exclude time spent
+     * reporting its start from its creation time.
+     */
+    restartTimer(): void;
+    /**
+     * Marks the completion of the index passed to the last call to
+     * {@link start}, returning the milliseconds it took to create.
+     */
+    finish(): number;
+    /** The `extraState` for {@link ReplicationStatusPublisher.publish}. */
+    readonly state: () => Partial<ReplicationState>;
+    status(): IndexingStatus | undefined;
+}
 export declare function publishReplicationError(lc: LogContext, stage: ReplicationStage, description: string, errorDetails?: JSONObject, now?: Date): Promise<void>;
 export declare function replicationStatusError(lc: LogContext, stage: ReplicationStage, e: unknown, db?: Database, now?: Date): ReplicationStatusEvent;
 export declare function replicationStatusEvent(lc: LogContext, db: Database | undefined, stage: ReplicationStage, status: Status, description?: string, now?: Date): ReplicationStatusEvent;

package/out/zero-cache/src/services/replicator/replication-status.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replication-status.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,qCAAqC,CAAC;AACpE,OAAO,KAAK,EAGV,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,MAAM,EACP,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAGtD,OAAO,EAEL,oBAAoB,EACrB,MAAM,+BAA+B,CAAC;AAKvC,qBAAa,0BAA0B;;IAKrC,MAAM,CAAC,UAAU,+BAEhB;IAED,MAAM,CAAC,qBAAqB,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,8BAAuB,8BAE1E;IAED,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,8BAAuB,8BASnE;IAED,YACE,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,CAAC,KAAK,CAAC,EAC3D,SAAS,8BAAuB,EAIjC;IAED,OAAO,CACL,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,QAAQ,SAAI,EACZ,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,EAC5C,GAAG,OAAa,GACf,IAAI,CAqBN;IAEK,oBAAoB,CACxB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,CAAC,EAAE,OAAO,GACT,OAAO,CAAC,KAAK,CAAC,CAOhB;IAED,IAAI,IAAI,IAAI,CAGX;CACF;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,WAAW,EAAE,MAAM,EACnB,YAAY,CAAC,EAAE,UAAU,EACzB,GAAG,OAAa,iBAYjB;AAED,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,CAAC,EAAE,OAAO,EACV,EAAE,CAAC,EAAE,QAAQ,EACb,GAAG,OAAa,0BAKjB;AAGD,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,GAAG,SAAS,EACxB,KAAK,EAAE,gBAAgB,EACvB,MAAM,EAAE,MAAM,EACd,WAAW,CAAC,EAAE,MAAM,EACpB,GAAG,OAAa,GACf,sBAAsB,CAmCxB"}
+{"version":3,"file":"replication-status.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,qCAAqC,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,MAAM,EACP,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAEtD,OAAO,KAAK,EAAC,aAAa,EAAgB,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAEL,oBAAoB,EACrB,MAAM,+BAA+B,CAAC;AAKvC,qBAAa,0BAA0B;;IAKrC,MAAM,CAAC,UAAU,+BAEhB;IAED,MAAM,CAAC,qBAAqB,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,8BAAuB,8BAE1E;IAED,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,8BAAuB,8BASnE;IAED,YACE,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,CAAC,KAAK,CAAC,EAC3D,SAAS,8BAAuB,EAIjC;IAED,OAAO,CACL,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,QAAQ,SAAI,EACZ,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,EAC5C,GAAG,OAAa,GACf,IAAI,CAGN;IAED;;;;;OAKG;IACG,eAAe,CACnB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,QAAQ,SAAI,EACZ,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,EAC5C,SAAS,SAAO,GACf,OAAO,CAAC,IAAI,CAAC,CAmBf;IAgCK,oBAAoB,CACxB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,CAAC,EAAE,OAAO,GACT,OAAO,CAAC,KAAK,CAAC,CAOhB;IAED,IAAI,IAAI,IAAI,CAGX;CACF;AAED;;;GAGG;AACH,qBAAa,gBAAgB;;IAS3B;;OAEG;IACH,YAAY,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,eAA0B,EAG/D;IAED,+CAA+C;IAC/C,KAAK,CAAC,KAAK,EAAE,aAAa,QAGzB;IAED;;;OAGG;IACH,YAAY,SAIX;IAED;;;OAGG;IACH,MAAM,IAAI,MAAM,CAUf;IAED,uEAAuE;IACvE,QAAQ,CAAC,KAAK,QAAO,OAAO,CAAC,gBAAgB,CAAC,CAG5C;IAEF,MAAM,IAAI,cAAc,GAAG,SAAS,CAoBnC;CACF;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,WAAW,EAAE,MAAM,EACnB,YAAY,CAAC,EAAE,UAAU,EACzB,GAAG,OAAa,iBAYjB;AAED,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,CAAC,EAAE,OAAO,EACV,EAAE,CAAC,EAAE,QAAQ,EACb,GAAG,OAAa,0BAKjB;AAGD,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,GAAG,SAAS,EACxB,KAAK,EAAE,gBAAgB,EACvB,MAAM,EAAE,MAAM,EACd,WAAW,CAAC,EAAE,MAAM,EACpB,GAAG,OAAa,GACf,sBAAsB,CAmCxB"}

package/out/zero-cache/src/services/replicator/replication-status.js

@@ -6,7 +6,7 @@ import { createSilentLogContext } from "../../../../shared/src/logging-test-util
 var byKeys = (a, b) => a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;
 var ReplicationStatusPublisher = class ReplicationStatusPublisher {
 	#dbRunner;
-	#publish;
+	#publishFn;
 	#timer;
 	static forTesting() {
 		return ReplicationStatusPublisher.forReplicaFile(":memory:");
@@ -26,23 +26,44 @@ var ReplicationStatusPublisher = class ReplicationStatusPublisher {
 	}
 	constructor(dbRunner, publishFn = publishCriticalEvent) {
 		this.#dbRunner = dbRunner;
-		this.#publish = publishFn;
+		this.#publishFn = publishFn;
 	}
 	publish(lc, stage, description, interval = 0, extraState, now = /* @__PURE__ */ new Date()) {
+		this.#publish(lc, stage, description, interval, extraState, now);
+		return this;
+	}
+	/**
+	* Like {@link publish}, but waits (for at most `maxWaitMs`) for the event
+	* to be sent. Use this before a long synchronous operation (e.g. creating
+	* an index) that blocks the event loop and would otherwise prevent the
+	* event from being sent until the operation completes.
+	*/
+	async publishAndFlush(lc, stage, description, interval = 0, extraState, maxWaitMs = 1e3) {
+		const published = this.#publish(lc, stage, description, interval, extraState);
+		let timer;
+		try {
+			await Promise.race([published, new Promise((resolve) => {
+				timer = setTimeout(resolve, maxWaitMs);
+			})]);
+		} finally {
+			clearTimeout(timer);
+		}
+	}
+	#publish(lc, stage, description, interval, extraState, now = /* @__PURE__ */ new Date()) {
 		this.stop();
 		const event = this.#dbRunner(lc, (db) => replicationStatusEvent(lc, db, stage, "OK", description, now));
 		if (event.state) event.state = {
 			...event.state,
 			...extraState?.()
 		};
-		this.#publish(lc, event);
+		const published = this.#publishFn(lc, event);
 		if (interval) this.#timer = setInterval(() => this.publish(lc, stage, description, interval, extraState), interval);
-		return this;
+		return published;
 	}
 	async publishAndThrowError(lc, stage, e) {
 		this.stop();
 		const event = this.#dbRunner(lc, (db) => replicationStatusError(lc, stage, e, db));
-		await this.#publish(lc, event);
+		await this.#publishFn(lc, event);
 		throw e;
 	}
 	stop() {
@@ -50,6 +71,75 @@ var ReplicationStatusPublisher = class ReplicationStatusPublisher {
 		return this;
 	}
 };
+/**
+* Tracks the progress of a sequence of index creations for reporting in
+* the `indexingStatus` field of {@link ReplicationState}.
+*/
+var IndexingProgress = class {
+	#totalIndexes;
+	#now;
+	#count = 0;
+	#completedMs = 0;
+	#current;
+	/**
+	* @param totalIndexes The total number of indexes to be created, if known.
+	*/
+	constructor(totalIndexes, now = () => performance.now()) {
+		this.#totalIndexes = totalIndexes;
+		this.#now = now;
+	}
+	/** Marks the start of creating the `index`. */
+	start(index) {
+		this.#count++;
+		this.#current = {
+			index,
+			start: this.#now(),
+			elapsedMs: void 0
+		};
+	}
+	/**
+	* Restarts the timer of the current index, e.g. to exclude time spent
+	* reporting its start from its creation time.
+	*/
+	restartTimer() {
+		if (this.#current) this.#current.start = this.#now();
+	}
+	/**
+	* Marks the completion of the index passed to the last call to
+	* {@link start}, returning the milliseconds it took to create.
+	*/
+	finish() {
+		const current = this.#current;
+		if (!current) return 0;
+		if (current.elapsedMs === void 0) {
+			current.elapsedMs = this.#now() - current.start;
+			this.#completedMs += current.elapsedMs;
+		}
+		return current.elapsedMs;
+	}
+	/** The `extraState` for {@link ReplicationStatusPublisher.publish}. */
+	state = () => {
+		const status = this.status();
+		return status ? { indexingStatus: status } : {};
+	};
+	status() {
+		const current = this.#current;
+		if (!current) return;
+		const { index, start, elapsedMs } = current;
+		const done = elapsedMs !== void 0;
+		return {
+			name: index.name,
+			table: index.tableName,
+			columns: Object.keys(index.columns),
+			unique: index.unique,
+			index: this.#count,
+			totalIndexes: this.#totalIndexes,
+			elapsedMs: Math.round(done ? elapsedMs : this.#now() - start),
+			completedMs: Math.round(done ? this.#completedMs - elapsedMs : this.#completedMs),
+			done
+		};
+	}
+};
 async function publishReplicationError(lc, stage, description, errorDetails, now = /* @__PURE__ */ new Date()) {
 	await publishCriticalEvent(lc, {
 		type: "zero/events/status/replication/v1",
@@ -130,6 +220,6 @@ function getReplicaSize(db) {
 	return pageCount * pageSize;
 }
 //#endregion
-export { ReplicationStatusPublisher, publishReplicationError, replicationStatusError };
+export { IndexingProgress, ReplicationStatusPublisher, publishReplicationError, replicationStatusError };
 
 //# sourceMappingURL=replication-status.js.map

package/out/zero-cache/src/services/replicator/replication-status.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-status.js","names":["#dbRunner","#publish","#timer"],"sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {createSilentLogContext} from '../../../../shared/src/logging-test-utils.ts';\nimport type {JSONObject} from '../../../../zero-events/src/json.ts';\nimport type {\n  ReplicatedIndex,\n  ReplicatedTable,\n  ReplicationStage,\n  ReplicationState,\n  ReplicationStatusEvent,\n  Status,\n} from '../../../../zero-events/src/status.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs, listIndexes} from '../../db/lite-tables.ts';\nimport type {LiteTableSpec} from '../../db/specs.ts';\nimport {\n  makeErrorDetails,\n  publishCriticalEvent,\n} from '../../observability/events.ts';\n\nconst byKeys = (a: [string, unknown], b: [string, unknown]) =>\n  a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;\n\nexport class ReplicationStatusPublisher {\n  readonly #dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T;\n  readonly #publish: typeof publishCriticalEvent;\n  #timer: NodeJS.Timeout | undefined;\n\n  static forTesting() {\n    return ReplicationStatusPublisher.forReplicaFile(':memory:');\n  }\n\n  static forRunningTransaction(tx: Database, publishFn = publishCriticalEvent) {\n    return new ReplicationStatusPublisher((_lc, fn) => fn(tx), publishFn);\n  }\n\n  static forReplicaFile(file: string, publishFn = publishCriticalEvent) {\n    return new ReplicationStatusPublisher((lc, fn) => {\n      const db = new Database(lc, file, {readonly: true});\n      try {\n        return fn(db);\n      } finally {\n        db.close();\n      }\n    }, publishFn);\n  }\n\n  constructor(\n    dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T,\n    publishFn = publishCriticalEvent,\n  ) {\n    this.#dbRunner = dbRunner;\n    this.#publish = publishFn;\n  }\n\n  publish(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description?: string,\n    interval = 0,\n    extraState?: () => Partial<ReplicationState>,\n    now = new Date(),\n  ): this {\n    this.stop();\n\n    const event = this.#dbRunner(lc, db =>\n      replicationStatusEvent(lc, db, stage, 'OK', description, now),\n    );\n    if (event.state) {\n      event.state = {\n        ...event.state,\n        ...extraState?.(),\n      };\n    }\n    void this.#publish(lc, event);\n\n    if (interval) {\n      this.#timer = setInterval(\n        () => this.publish(lc, stage, description, interval, extraState),\n        interval,\n      );\n    }\n    return this;\n  }\n\n  async publishAndThrowError(\n    lc: LogContext,\n    stage: ReplicationStage,\n    e: unknown,\n  ): Promise<never> {\n    this.stop();\n    const event = this.#dbRunner(lc, db =>\n      replicationStatusError(lc, stage, e, db),\n    );\n    await this.#publish(lc, event);\n    throw e;\n  }\n\n  stop(): this {\n    clearInterval(this.#timer);\n    return this;\n  }\n}\n\nexport async function publishReplicationError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  description: string,\n  errorDetails?: JSONObject,\n  now = new Date(),\n) {\n  const event: ReplicationStatusEvent = {\n    type: 'zero/events/status/replication/v1',\n    component: 'replication',\n    status: 'ERROR',\n    stage,\n    description,\n    errorDetails,\n    time: now.toISOString(),\n  };\n  await publishCriticalEvent(lc, event);\n}\n\nexport function replicationStatusError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  e: unknown,\n  db?: Database,\n  now = new Date(),\n) {\n  const event = replicationStatusEvent(lc, db, stage, 'ERROR', String(e), now);\n  event.errorDetails = makeErrorDetails(e);\n  return event;\n}\n\n// Exported for testing.\nexport function replicationStatusEvent(\n  lc: LogContext,\n  db: Database | undefined,\n  stage: ReplicationStage,\n  status: Status,\n  description?: string,\n  now = new Date(),\n): ReplicationStatusEvent {\n  const start = performance.now();\n  try {\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: db ? getReplicatedTables(db) : [],\n        indexes: db ? getReplicatedIndexes(db) : [],\n        replicaSize: db ? getReplicaSize(db) : undefined,\n      },\n    };\n  } catch (e) {\n    lc.warn?.(`Unable to create full ReplicationStatusEvent`, e);\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: [],\n        indexes: [],\n        replicaSize: 0,\n      },\n    };\n  } finally {\n    const elapsed = (performance.now() - start).toFixed(3);\n    lc.debug?.(`computed schema for replication event (${elapsed} ms)`);\n  }\n}\n\nfunction getReplicatedTables(db: Database): ReplicatedTable[] {\n  const fullTables = new Map<string, LiteTableSpec>();\n  const clientSchema = computeZqlSpecs(\n    createSilentLogContext(), // avoid logging warnings about indexes\n    db,\n    {includeBackfillingColumns: false},\n    new Map(),\n    fullTables,\n  );\n\n  // oxlint-disable-next-line e18e/prefer-array-to-sorted\n  return [...fullTables.entries()].sort(byKeys).map(([table, spec]) => ({\n    table,\n    columns: Object.entries(spec.columns)\n      .sort(byKeys)\n      .map(([column, spec]) => ({\n        column,\n        upstreamType: spec.dataType.split('|')[0],\n        clientType: clientSchema.get(table)?.zqlSpec[column]?.type ?? null,\n      })),\n  }));\n}\n\nfunction getReplicatedIndexes(db: Database): ReplicatedIndex[] {\n  return listIndexes(db).map(({tableName: table, columns, unique}) => ({\n    table,\n    unique,\n    columns: Object.entries(columns)\n      .sort(byKeys)\n      .map(([column, dir]) => ({column, dir})),\n  }));\n}\n\nfunction getReplicaSize(db: Database) {\n  const [{page_count: pageCount}] = db.pragma<{page_count: number}>(\n    'page_count',\n  );\n  const [{page_size: pageSize}] = db.pragma<{page_size: number}>('page_size');\n  return pageCount * pageSize;\n}\n"],"mappings":";;;;;AAmBA,IAAM,UAAU,GAAsB,MACpC,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI;AAEvC,IAAa,6BAAb,MAAa,2BAA2B;CACtC;CACA;CACA;CAEA,OAAO,aAAa;EAClB,OAAO,2BAA2B,eAAe,UAAU;CAC7D;CAEA,OAAO,sBAAsB,IAAc,YAAY,sBAAsB;EAC3E,OAAO,IAAI,4BAA4B,KAAK,OAAO,GAAG,EAAE,GAAG,SAAS;CACtE;CAEA,OAAO,eAAe,MAAc,YAAY,sBAAsB;EACpE,OAAO,IAAI,4BAA4B,IAAI,OAAO;GAChD,MAAM,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;GAClD,IAAI;IACF,OAAO,GAAG,EAAE;GACd,UAAU;IACR,GAAG,MAAM;GACX;EACF,GAAG,SAAS;CACd;CAEA,YACE,UACA,YAAY,sBACZ;EACA,KAAKA,YAAY;EACjB,KAAKC,WAAW;CAClB;CAEA,QACE,IACA,OACA,aACA,WAAW,GACX,YACA,sBAAM,IAAI,KAAK,GACT;EACN,KAAK,KAAK;EAEV,MAAM,QAAQ,KAAKD,UAAU,KAAI,OAC/B,uBAAuB,IAAI,IAAI,OAAO,MAAM,aAAa,GAAG,CAC9D;EACA,IAAI,MAAM,OACR,MAAM,QAAQ;GACZ,GAAG,MAAM;GACT,GAAG,aAAa;EAClB;EAEF,KAAUC,SAAS,IAAI,KAAK;EAE5B,IAAI,UACF,KAAKC,SAAS,kBACN,KAAK,QAAQ,IAAI,OAAO,aAAa,UAAU,UAAU,GAC/D,QACF;EAEF,OAAO;CACT;CAEA,MAAM,qBACJ,IACA,OACA,GACgB;EAChB,KAAK,KAAK;EACV,MAAM,QAAQ,KAAKF,UAAU,KAAI,OAC/B,uBAAuB,IAAI,OAAO,GAAG,EAAE,CACzC;EACA,MAAM,KAAKC,SAAS,IAAI,KAAK;EAC7B,MAAM;CACR;CAEA,OAAa;EACX,cAAc,KAAKC,MAAM;EACzB,OAAO;CACT;AACF;AAEA,eAAsB,wBACpB,IACA,OACA,aACA,cACA,sBAAM,IAAI,KAAK,GACf;CAUA,MAAM,qBAAqB,IAAI;EAR7B,MAAM;EACN,WAAW;EACX,QAAQ;EACR;EACA;EACA;EACA,MAAM,IAAI,YAAY;CAEO,CAAK;AACtC;AAEA,SAAgB,uBACd,IACA,OACA,GACA,IACA,sBAAM,IAAI,KAAK,GACf;CACA,MAAM,QAAQ,uBAAuB,IAAI,IAAI,OAAO,SAAS,OAAO,CAAC,GAAG,GAAG;CAC3E,MAAM,eAAe,iBAAiB,CAAC;CACvC,OAAO;AACT;AAGA,SAAgB,uBACd,IACA,IACA,OACA,QACA,aACA,sBAAM,IAAI,KAAK,GACS;CACxB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;EACF,OAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,YAAY;GACtB,OAAO;IACL,QAAQ,KAAK,oBAAoB,EAAE,IAAI,CAAC;IACxC,SAAS,KAAK,qBAAqB,EAAE,IAAI,CAAC;IAC1C,aAAa,KAAK,eAAe,EAAE,IAAI,KAAA;GACzC;EACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,gDAAgD,CAAC;EAC3D,OAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,YAAY;GACtB,OAAO;IACL,QAAQ,CAAC;IACT,SAAS,CAAC;IACV,aAAa;GACf;EACF;CACF,UAAU;EACR,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,GAAG,QAAQ,0CAA0C,QAAQ,KAAK;CACpE;AACF;AAEA,SAAS,oBAAoB,IAAiC;CAC5D,MAAM,6BAAa,IAAI,IAA2B;CAClD,MAAM,eAAe,gBACnB,uBAAuB,GACvB,IACA,EAAC,2BAA2B,MAAK,mBACjC,IAAI,IAAI,GACR,UACF;CAGA,OAAO,CAAC,GAAG,WAAW,QAAQ,CAAC,EAAE,KAAK,MAAM,EAAE,KAAK,CAAC,OAAO,WAAW;EACpE;EACA,SAAS,OAAO,QAAQ,KAAK,OAAO,EACjC,KAAK,MAAM,EACX,KAAK,CAAC,QAAQ,WAAW;GACxB;GACA,cAAc,KAAK,SAAS,MAAM,GAAG,EAAE;GACvC,YAAY,aAAa,IAAI,KAAK,GAAG,QAAQ,SAAS,QAAQ;EAChE,EAAE;CACN,EAAE;AACJ;AAEA,SAAS,qBAAqB,IAAiC;CAC7D,OAAO,YAAY,EAAE,EAAE,KAAK,EAAC,WAAW,OAAO,SAAS,cAAa;EACnE;EACA;EACA,SAAS,OAAO,QAAQ,OAAO,EAC5B,KAAK,MAAM,EACX,KAAK,CAAC,QAAQ,UAAU;GAAC;GAAQ;EAAG,EAAE;CAC3C,EAAE;AACJ;AAEA,SAAS,eAAe,IAAc;CACpC,MAAM,CAAC,EAAC,YAAY,eAAc,GAAG,OACnC,YACF;CACA,MAAM,CAAC,EAAC,WAAW,cAAa,GAAG,OAA4B,WAAW;CAC1E,OAAO,YAAY;AACrB"}
+{"version":3,"file":"replication-status.js","names":["#dbRunner","#publishFn","#publish","#timer","#totalIndexes","#now","#count","#current","#completedMs"],"sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {createSilentLogContext} from '../../../../shared/src/logging-test-utils.ts';\nimport type {JSONObject} from '../../../../zero-events/src/json.ts';\nimport type {\n  IndexingStatus,\n  ReplicatedIndex,\n  ReplicatedTable,\n  ReplicationStage,\n  ReplicationState,\n  ReplicationStatusEvent,\n  Status,\n} from '../../../../zero-events/src/status.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs, listIndexes} from '../../db/lite-tables.ts';\nimport type {LiteIndexSpec, LiteTableSpec} from '../../db/specs.ts';\nimport {\n  makeErrorDetails,\n  publishCriticalEvent,\n} from '../../observability/events.ts';\n\nconst byKeys = (a: [string, unknown], b: [string, unknown]) =>\n  a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;\n\nexport class ReplicationStatusPublisher {\n  readonly #dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T;\n  readonly #publishFn: typeof publishCriticalEvent;\n  #timer: NodeJS.Timeout | undefined;\n\n  static forTesting() {\n    return ReplicationStatusPublisher.forReplicaFile(':memory:');\n  }\n\n  static forRunningTransaction(tx: Database, publishFn = publishCriticalEvent) {\n    return new ReplicationStatusPublisher((_lc, fn) => fn(tx), publishFn);\n  }\n\n  static forReplicaFile(file: string, publishFn = publishCriticalEvent) {\n    return new ReplicationStatusPublisher((lc, fn) => {\n      const db = new Database(lc, file, {readonly: true});\n      try {\n        return fn(db);\n      } finally {\n        db.close();\n      }\n    }, publishFn);\n  }\n\n  constructor(\n    dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T,\n    publishFn = publishCriticalEvent,\n  ) {\n    this.#dbRunner = dbRunner;\n    this.#publishFn = publishFn;\n  }\n\n  publish(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description?: string,\n    interval = 0,\n    extraState?: () => Partial<ReplicationState>,\n    now = new Date(),\n  ): this {\n    void this.#publish(lc, stage, description, interval, extraState, now);\n    return this;\n  }\n\n  /**\n   * Like {@link publish}, but waits (for at most `maxWaitMs`) for the event\n   * to be sent. Use this before a long synchronous operation (e.g. creating\n   * an index) that blocks the event loop and would otherwise prevent the\n   * event from being sent until the operation completes.\n   */\n  async publishAndFlush(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description?: string,\n    interval = 0,\n    extraState?: () => Partial<ReplicationState>,\n    maxWaitMs = 1000,\n  ): Promise<void> {\n    const published = this.#publish(\n      lc,\n      stage,\n      description,\n      interval,\n      extraState,\n    );\n    let timer: NodeJS.Timeout | undefined;\n    try {\n      await Promise.race([\n        published,\n        new Promise<void>(resolve => {\n          timer = setTimeout(resolve, maxWaitMs);\n        }),\n      ]);\n    } finally {\n      clearTimeout(timer);\n    }\n  }\n\n  #publish(\n    lc: LogContext,\n    stage: ReplicationStage,\n    description: string | undefined,\n    interval: number,\n    extraState: (() => Partial<ReplicationState>) | undefined,\n    now = new Date(),\n  ): Promise<void> {\n    this.stop();\n\n    const event = this.#dbRunner(lc, db =>\n      replicationStatusEvent(lc, db, stage, 'OK', description, now),\n    );\n    if (event.state) {\n      event.state = {\n        ...event.state,\n        ...extraState?.(),\n      };\n    }\n    const published = this.#publishFn(lc, event);\n\n    if (interval) {\n      this.#timer = setInterval(\n        () => this.publish(lc, stage, description, interval, extraState),\n        interval,\n      );\n    }\n    return published;\n  }\n\n  async publishAndThrowError(\n    lc: LogContext,\n    stage: ReplicationStage,\n    e: unknown,\n  ): Promise<never> {\n    this.stop();\n    const event = this.#dbRunner(lc, db =>\n      replicationStatusError(lc, stage, e, db),\n    );\n    await this.#publishFn(lc, event);\n    throw e;\n  }\n\n  stop(): this {\n    clearInterval(this.#timer);\n    return this;\n  }\n}\n\n/**\n * Tracks the progress of a sequence of index creations for reporting in\n * the `indexingStatus` field of {@link ReplicationState}.\n */\nexport class IndexingProgress {\n  readonly #totalIndexes: number | undefined;\n  readonly #now: () => number;\n  #count = 0;\n  #completedMs = 0;\n  #current:\n    | {index: LiteIndexSpec; start: number; elapsedMs: number | undefined}\n    | undefined;\n\n  /**\n   * @param totalIndexes The total number of indexes to be created, if known.\n   */\n  constructor(totalIndexes?: number, now = () => performance.now()) {\n    this.#totalIndexes = totalIndexes;\n    this.#now = now;\n  }\n\n  /** Marks the start of creating the `index`. */\n  start(index: LiteIndexSpec) {\n    this.#count++;\n    this.#current = {index, start: this.#now(), elapsedMs: undefined};\n  }\n\n  /**\n   * Restarts the timer of the current index, e.g. to exclude time spent\n   * reporting its start from its creation time.\n   */\n  restartTimer() {\n    if (this.#current) {\n      this.#current.start = this.#now();\n    }\n  }\n\n  /**\n   * Marks the completion of the index passed to the last call to\n   * {@link start}, returning the milliseconds it took to create.\n   */\n  finish(): number {\n    const current = this.#current;\n    if (!current) {\n      return 0;\n    }\n    if (current.elapsedMs === undefined) {\n      current.elapsedMs = this.#now() - current.start;\n      this.#completedMs += current.elapsedMs;\n    }\n    return current.elapsedMs;\n  }\n\n  /** The `extraState` for {@link ReplicationStatusPublisher.publish}. */\n  readonly state = (): Partial<ReplicationState> => {\n    const status = this.status();\n    return status ? {indexingStatus: status} : {};\n  };\n\n  status(): IndexingStatus | undefined {\n    const current = this.#current;\n    if (!current) {\n      return undefined;\n    }\n    const {index, start, elapsedMs} = current;\n    const done = elapsedMs !== undefined;\n    return {\n      name: index.name,\n      table: index.tableName,\n      columns: Object.keys(index.columns),\n      unique: index.unique,\n      index: this.#count,\n      totalIndexes: this.#totalIndexes,\n      elapsedMs: Math.round(done ? elapsedMs : this.#now() - start),\n      completedMs: Math.round(\n        done ? this.#completedMs - elapsedMs : this.#completedMs,\n      ),\n      done,\n    };\n  }\n}\n\nexport async function publishReplicationError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  description: string,\n  errorDetails?: JSONObject,\n  now = new Date(),\n) {\n  const event: ReplicationStatusEvent = {\n    type: 'zero/events/status/replication/v1',\n    component: 'replication',\n    status: 'ERROR',\n    stage,\n    description,\n    errorDetails,\n    time: now.toISOString(),\n  };\n  await publishCriticalEvent(lc, event);\n}\n\nexport function replicationStatusError(\n  lc: LogContext,\n  stage: ReplicationStage,\n  e: unknown,\n  db?: Database,\n  now = new Date(),\n) {\n  const event = replicationStatusEvent(lc, db, stage, 'ERROR', String(e), now);\n  event.errorDetails = makeErrorDetails(e);\n  return event;\n}\n\n// Exported for testing.\nexport function replicationStatusEvent(\n  lc: LogContext,\n  db: Database | undefined,\n  stage: ReplicationStage,\n  status: Status,\n  description?: string,\n  now = new Date(),\n): ReplicationStatusEvent {\n  const start = performance.now();\n  try {\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: db ? getReplicatedTables(db) : [],\n        indexes: db ? getReplicatedIndexes(db) : [],\n        replicaSize: db ? getReplicaSize(db) : undefined,\n      },\n    };\n  } catch (e) {\n    lc.warn?.(`Unable to create full ReplicationStatusEvent`, e);\n    return {\n      type: 'zero/events/status/replication/v1',\n      component: 'replication',\n      status,\n      stage,\n      description,\n      time: now.toISOString(),\n      state: {\n        tables: [],\n        indexes: [],\n        replicaSize: 0,\n      },\n    };\n  } finally {\n    const elapsed = (performance.now() - start).toFixed(3);\n    lc.debug?.(`computed schema for replication event (${elapsed} ms)`);\n  }\n}\n\nfunction getReplicatedTables(db: Database): ReplicatedTable[] {\n  const fullTables = new Map<string, LiteTableSpec>();\n  const clientSchema = computeZqlSpecs(\n    createSilentLogContext(), // avoid logging warnings about indexes\n    db,\n    {includeBackfillingColumns: false},\n    new Map(),\n    fullTables,\n  );\n\n  // oxlint-disable-next-line e18e/prefer-array-to-sorted\n  return [...fullTables.entries()].sort(byKeys).map(([table, spec]) => ({\n    table,\n    columns: Object.entries(spec.columns)\n      .sort(byKeys)\n      .map(([column, spec]) => ({\n        column,\n        upstreamType: spec.dataType.split('|')[0],\n        clientType: clientSchema.get(table)?.zqlSpec[column]?.type ?? null,\n      })),\n  }));\n}\n\nfunction getReplicatedIndexes(db: Database): ReplicatedIndex[] {\n  return listIndexes(db).map(({tableName: table, columns, unique}) => ({\n    table,\n    unique,\n    columns: Object.entries(columns)\n      .sort(byKeys)\n      .map(([column, dir]) => ({column, dir})),\n  }));\n}\n\nfunction getReplicaSize(db: Database) {\n  const [{page_count: pageCount}] = db.pragma<{page_count: number}>(\n    'page_count',\n  );\n  const [{page_size: pageSize}] = db.pragma<{page_size: number}>('page_size');\n  return pageCount * pageSize;\n}\n"],"mappings":";;;;;AAoBA,IAAM,UAAU,GAAsB,MACpC,EAAE,KAAK,EAAE,KAAK,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI;AAEvC,IAAa,6BAAb,MAAa,2BAA2B;CACtC;CACA;CACA;CAEA,OAAO,aAAa;EAClB,OAAO,2BAA2B,eAAe,UAAU;CAC7D;CAEA,OAAO,sBAAsB,IAAc,YAAY,sBAAsB;EAC3E,OAAO,IAAI,4BAA4B,KAAK,OAAO,GAAG,EAAE,GAAG,SAAS;CACtE;CAEA,OAAO,eAAe,MAAc,YAAY,sBAAsB;EACpE,OAAO,IAAI,4BAA4B,IAAI,OAAO;GAChD,MAAM,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;GAClD,IAAI;IACF,OAAO,GAAG,EAAE;GACd,UAAU;IACR,GAAG,MAAM;GACX;EACF,GAAG,SAAS;CACd;CAEA,YACE,UACA,YAAY,sBACZ;EACA,KAAKA,YAAY;EACjB,KAAKC,aAAa;CACpB;CAEA,QACE,IACA,OACA,aACA,WAAW,GACX,YACA,sBAAM,IAAI,KAAK,GACT;EACN,KAAUC,SAAS,IAAI,OAAO,aAAa,UAAU,YAAY,GAAG;EACpE,OAAO;CACT;;;;;;;CAQA,MAAM,gBACJ,IACA,OACA,aACA,WAAW,GACX,YACA,YAAY,KACG;EACf,MAAM,YAAY,KAAKA,SACrB,IACA,OACA,aACA,UACA,UACF;EACA,IAAI;EACJ,IAAI;GACF,MAAM,QAAQ,KAAK,CACjB,WACA,IAAI,SAAc,YAAW;IAC3B,QAAQ,WAAW,SAAS,SAAS;GACvC,CAAC,CACH,CAAC;EACH,UAAU;GACR,aAAa,KAAK;EACpB;CACF;CAEA,SACE,IACA,OACA,aACA,UACA,YACA,sBAAM,IAAI,KAAK,GACA;EACf,KAAK,KAAK;EAEV,MAAM,QAAQ,KAAKF,UAAU,KAAI,OAC/B,uBAAuB,IAAI,IAAI,OAAO,MAAM,aAAa,GAAG,CAC9D;EACA,IAAI,MAAM,OACR,MAAM,QAAQ;GACZ,GAAG,MAAM;GACT,GAAG,aAAa;EAClB;EAEF,MAAM,YAAY,KAAKC,WAAW,IAAI,KAAK;EAE3C,IAAI,UACF,KAAKE,SAAS,kBACN,KAAK,QAAQ,IAAI,OAAO,aAAa,UAAU,UAAU,GAC/D,QACF;EAEF,OAAO;CACT;CAEA,MAAM,qBACJ,IACA,OACA,GACgB;EAChB,KAAK,KAAK;EACV,MAAM,QAAQ,KAAKH,UAAU,KAAI,OAC/B,uBAAuB,IAAI,OAAO,GAAG,EAAE,CACzC;EACA,MAAM,KAAKC,WAAW,IAAI,KAAK;EAC/B,MAAM;CACR;CAEA,OAAa;EACX,cAAc,KAAKE,MAAM;EACzB,OAAO;CACT;AACF;;;;;AAMA,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA,SAAS;CACT,eAAe;CACf;;;;CAOA,YAAY,cAAuB,YAAY,YAAY,IAAI,GAAG;EAChE,KAAKC,gBAAgB;EACrB,KAAKC,OAAO;CACd;;CAGA,MAAM,OAAsB;EAC1B,KAAKC;EACL,KAAKC,WAAW;GAAC;GAAO,OAAO,KAAKF,KAAK;GAAG,WAAW,KAAA;EAAS;CAClE;;;;;CAMA,eAAe;EACb,IAAI,KAAKE,UACP,KAAKA,SAAS,QAAQ,KAAKF,KAAK;CAEpC;;;;;CAMA,SAAiB;EACf,MAAM,UAAU,KAAKE;EACrB,IAAI,CAAC,SACH,OAAO;EAET,IAAI,QAAQ,cAAc,KAAA,GAAW;GACnC,QAAQ,YAAY,KAAKF,KAAK,IAAI,QAAQ;GAC1C,KAAKG,gBAAgB,QAAQ;EAC/B;EACA,OAAO,QAAQ;CACjB;;CAGA,cAAkD;EAChD,MAAM,SAAS,KAAK,OAAO;EAC3B,OAAO,SAAS,EAAC,gBAAgB,OAAM,IAAI,CAAC;CAC9C;CAEA,SAAqC;EACnC,MAAM,UAAU,KAAKD;EACrB,IAAI,CAAC,SACH;EAEF,MAAM,EAAC,OAAO,OAAO,cAAa;EAClC,MAAM,OAAO,cAAc,KAAA;EAC3B,OAAO;GACL,MAAM,MAAM;GACZ,OAAO,MAAM;GACb,SAAS,OAAO,KAAK,MAAM,OAAO;GAClC,QAAQ,MAAM;GACd,OAAO,KAAKD;GACZ,cAAc,KAAKF;GACnB,WAAW,KAAK,MAAM,OAAO,YAAY,KAAKC,KAAK,IAAI,KAAK;GAC5D,aAAa,KAAK,MAChB,OAAO,KAAKG,eAAe,YAAY,KAAKA,YAC9C;GACA;EACF;CACF;AACF;AAEA,eAAsB,wBACpB,IACA,OACA,aACA,cACA,sBAAM,IAAI,KAAK,GACf;CAUA,MAAM,qBAAqB,IAAI;EAR7B,MAAM;EACN,WAAW;EACX,QAAQ;EACR;EACA;EACA;EACA,MAAM,IAAI,YAAY;CAEO,CAAK;AACtC;AAEA,SAAgB,uBACd,IACA,OACA,GACA,IACA,sBAAM,IAAI,KAAK,GACf;CACA,MAAM,QAAQ,uBAAuB,IAAI,IAAI,OAAO,SAAS,OAAO,CAAC,GAAG,GAAG;CAC3E,MAAM,eAAe,iBAAiB,CAAC;CACvC,OAAO;AACT;AAGA,SAAgB,uBACd,IACA,IACA,OACA,QACA,aACA,sBAAM,IAAI,KAAK,GACS;CACxB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;EACF,OAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,YAAY;GACtB,OAAO;IACL,QAAQ,KAAK,oBAAoB,EAAE,IAAI,CAAC;IACxC,SAAS,KAAK,qBAAqB,EAAE,IAAI,CAAC;IAC1C,aAAa,KAAK,eAAe,EAAE,IAAI,KAAA;GACzC;EACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,gDAAgD,CAAC;EAC3D,OAAO;GACL,MAAM;GACN,WAAW;GACX;GACA;GACA;GACA,MAAM,IAAI,YAAY;GACtB,OAAO;IACL,QAAQ,CAAC;IACT,SAAS,CAAC;IACV,aAAa;GACf;EACF;CACF,UAAU;EACR,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;EACrD,GAAG,QAAQ,0CAA0C,QAAQ,KAAK;CACpE;AACF;AAEA,SAAS,oBAAoB,IAAiC;CAC5D,MAAM,6BAAa,IAAI,IAA2B;CAClD,MAAM,eAAe,gBACnB,uBAAuB,GACvB,IACA,EAAC,2BAA2B,MAAK,mBACjC,IAAI,IAAI,GACR,UACF;CAGA,OAAO,CAAC,GAAG,WAAW,QAAQ,CAAC,EAAE,KAAK,MAAM,EAAE,KAAK,CAAC,OAAO,WAAW;EACpE;EACA,SAAS,OAAO,QAAQ,KAAK,OAAO,EACjC,KAAK,MAAM,EACX,KAAK,CAAC,QAAQ,WAAW;GACxB;GACA,cAAc,KAAK,SAAS,MAAM,GAAG,EAAE;GACvC,YAAY,aAAa,IAAI,KAAK,GAAG,QAAQ,SAAS,QAAQ;EAChE,EAAE;CACN,EAAE;AACJ;AAEA,SAAS,qBAAqB,IAAiC;CAC7D,OAAO,YAAY,EAAE,EAAE,KAAK,EAAC,WAAW,OAAO,SAAS,cAAa;EACnE;EACA;EACA,SAAS,OAAO,QAAQ,OAAO,EAC5B,KAAK,MAAM,EACX,KAAK,CAAC,QAAQ,UAAU;GAAC;GAAQ;EAAG,EAAE;CAC3C,EAAE;AACJ;AAEA,SAAS,eAAe,IAAc;CACpC,MAAM,CAAC,EAAC,YAAY,eAAc,GAAG,OACnC,YACF;CACA,MAAM,CAAC,EAAC,WAAW,cAAa,GAAG,OAA4B,WAAW;CAC1E,OAAO,YAAY;AACrB"}

package/out/zero-cache/src/services/replicator/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":["#lc","#incrementalSyncer","#shouldPublishErrors","#worker","#runPromise"],"sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {ReadonlyJSONObject} from '../../../../shared/src/json.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {ChangeStreamer} from '../change-streamer/change-streamer.ts';\nimport type {Service} from '../service.ts';\nimport {IncrementalSyncer} from './incremental-sync.ts';\nimport {\n  publishReplicationError,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/** See {@link ReplicaStateNotifier.subscribe()}. */\nexport type ReplicaState = {\n  readonly state: 'version-ready';\n  /**\n   * The replica watermark that became ready. Omitted for the initial\n   * \"current state is ready\" notification, which should not count as newly\n   * unserved work for a freshly-started ViewSyncer.\n   */\n  readonly watermark?: string | undefined;\n  /**\n   * Millisecond epoch when `watermark` became ready to read from the replica.\n   */\n  readonly replicaReadyTimeMs?: number | undefined;\n\n  /**\n   * Millisecond epoch at which the transaction behind `watermark` committed\n   * *upstream*, as reported by the ChangeSource. Unlike `replicaReadyTimeMs`,\n   * which is stamped locally when the change lands in the replica, this is the\n   * origin of the whole pipeline, so `now - upstreamCommitTimeMs` at the point\n   * the change is poked to clients is the end-to-end serving lag.\n   *\n   * Note that this crosses a clock domain: it comes from the upstream\n   * database, whereas it is subtracted from the local clock in the ViewSyncer.\n   * The existing `zero.replication.total_lag` metric spans the same boundary.\n   *\n   * Undefined when the ChangeSource does not report a commit time, and for the\n   * initial \"current state is ready\" notification.\n   */\n  readonly upstreamCommitTimeMs?: number | undefined;\n\n  // Used in tests to verify behavior when additional information\n  // is ferried in the future. Not set in production.\n  readonly testSeqNum?: number | undefined;\n};\n\nexport interface ReplicaStateNotifier {\n  /**\n   * Creates a cancelable subscription of changes in the replica state.\n   *\n   * A `version-ready` message indicates that the replica is ready to be\n   * read, and henceforth that a _new_ version is ready, i.e. whenever a\n   * change is committed to the replica. The `version-ready` message itself\n   * otherwise contains no other information; the subscriber queries the\n   * replica for the current data.\n   *\n   * A `maintenance` state indicates that the replica should not be read from.\n   * If a subscriber is holding any transaction locks, it should release them\n   * until the next `version-ready` signal.\n   *\n   * Upon subscription, the current state of the replica is sent immediately\n   * if known. If multiple notifications occur before the subscriber\n   * can consume them, all but the last notification are discarded by the\n   * Subscription object (i.e. not buffered). Thus, a subscriber only\n   * ever consumes the current (i.e. known) state of the replica. This avoids\n   * a buildup of \"work\" if a subscriber is too busy to consume all\n   * notifications.\n   */\n  subscribe(): Source<ReplicaState>;\n}\n\nexport interface Replicator extends ReplicaStateNotifier {\n  /**\n   * Returns an opaque message for human-readable consumption. This is\n   * purely for ensuring that the Replicator has started at least once to\n   * bootstrap a new replica.\n   */\n  status(): Promise<ReadonlyJSONObject>;\n}\n\nexport type ReplicatorMode = 'backup' | 'serving';\n\nexport class ReplicatorService implements Replicator, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #incrementalSyncer: IncrementalSyncer;\n  readonly #shouldPublishErrors: boolean;\n  readonly #worker: WriteWorkerClient;\n  #runPromise: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    mode: ReplicatorMode,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    statusPublisher: ReplicationStatusPublisher | null,\n  ) {\n    this.id = id;\n    this.#lc = lc\n      .withContext('component', 'replicator')\n      .withContext('serviceID', this.id);\n    this.#worker = worker;\n    this.#shouldPublishErrors = statusPublisher !== null;\n\n    this.#incrementalSyncer = new IncrementalSyncer(\n      this.#lc,\n      taskID,\n      `${taskID}/${id}`,\n      changeStreamer,\n      worker,\n      mode,\n      statusPublisher,\n    );\n  }\n\n  status() {\n    return Promise.resolve({status: 'ok'});\n  }\n\n  run() {\n    this.#runPromise = this.#incrementalSyncer.run().catch(async error => {\n      if (this.#shouldPublishErrors) {\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          'Replication stopped because the replica writer failed',\n        );\n      }\n      throw error;\n    });\n    return this.#runPromise;\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#incrementalSyncer.subscribe();\n  }\n\n  async stop() {\n    this.#incrementalSyncer.stop(this.#lc);\n    // Wait for the syncer's run loop to finish so that any in-flight\n    // worker.processMessage() call completes and clears #pending\n    // before we send the 'stop' message to the worker.\n    await this.#runPromise;\n    await this.#worker.stop();\n  }\n}\n"],"mappings":";;;AAmFA,IAAa,oBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,IACA,MACA,gBACA,QACA,iBACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GACR,YAAY,aAAa,YAAY,EACrC,YAAY,aAAa,KAAK,EAAE;EACnC,KAAKG,UAAU;EACf,KAAKD,uBAAuB,oBAAoB;EAEhD,KAAKD,qBAAqB,IAAI,kBAC5B,KAAKD,KACL,QACA,GAAG,OAAO,GAAG,MACb,gBACA,QACA,MACA,eACF;CACF;CAEA,SAAS;EACP,OAAO,QAAQ,QAAQ,EAAC,QAAQ,KAAI,CAAC;CACvC;CAEA,MAAM;EACJ,KAAKI,cAAc,KAAKH,mBAAmB,IAAI,EAAE,MAAM,OAAM,UAAS;GACpE,IAAI,KAAKC,sBACP,MAAM,wBACJ,KAAKF,KACL,eACA,uDACF;GAEF,MAAM;EACR,CAAC;EACD,OAAO,KAAKI;CACd;CAEA,YAAkC;EAChC,OAAO,KAAKH,mBAAmB,UAAU;CAC3C;CAEA,MAAM,OAAO;EACX,KAAKA,mBAAmB,KAAK,KAAKD,GAAG;EAIrC,MAAM,KAAKI;EACX,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF"}
+{"version":3,"file":"replicator.js","names":["#lc","#incrementalSyncer","#shouldPublishErrors","#worker","#runPromise"],"sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {ReadonlyJSONObject} from '../../../../shared/src/json.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {ChangeStreamer} from '../change-streamer/change-streamer.ts';\nimport type {Service} from '../service.ts';\nimport {IncrementalSyncer} from './incremental-sync.ts';\nimport {\n  publishReplicationError,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/** See {@link ReplicaStateNotifier.subscribe()}. */\nexport type ReplicaState = {\n  readonly state: 'version-ready';\n  /**\n   * The replica watermark that became ready. Omitted for the initial\n   * \"current state is ready\" notification, which should not count as newly\n   * unserved work for a freshly-started ViewSyncer.\n   */\n  readonly watermark?: string | undefined;\n  /**\n   * Millisecond epoch when `watermark` became ready to read from the replica.\n   */\n  readonly replicaReadyTimeMs?: number | undefined;\n\n  /**\n   * Millisecond epoch at which the transaction behind `watermark` committed\n   * *upstream*, as reported by the ChangeSource. Unlike `replicaReadyTimeMs`,\n   * which is stamped locally when the change lands in the replica, this is the\n   * origin of the whole pipeline, so `now - upstreamCommitTimeMs` at the point\n   * the change is poked to clients is the end-to-end serving lag.\n   *\n   * Note that this crosses a clock domain: it comes from the upstream\n   * database, whereas it is subtracted from the local clock in the ViewSyncer.\n   * The existing `zero.replication.total_lag` metric spans the same boundary.\n   *\n   * Undefined when the ChangeSource does not report a commit time, and for the\n   * initial \"current state is ready\" notification.\n   */\n  readonly upstreamCommitTimeMs?: number | undefined;\n\n  // Used in tests to verify behavior when additional information\n  // is ferried in the future. Not set in production.\n  readonly testSeqNum?: number | undefined;\n};\n\nexport interface ReplicaStateNotifier {\n  /**\n   * Creates a cancelable subscription of changes in the replica state.\n   *\n   * A `version-ready` message indicates that the replica is ready to be\n   * read, and henceforth that a _new_ version is ready, i.e. whenever a\n   * change is committed to the replica. The `version-ready` message itself\n   * otherwise contains no other information; the subscriber queries the\n   * replica for the current data.\n   *\n   * A `maintenance` state indicates that the replica should not be read from.\n   * If a subscriber is holding any transaction locks, it should release them\n   * until the next `version-ready` signal.\n   *\n   * Upon subscription, the current state of the replica is sent immediately\n   * if known. If multiple notifications occur before the subscriber\n   * can consume them, all but the last notification are discarded by the\n   * Subscription object (i.e. not buffered). Thus, a subscriber only\n   * ever consumes the current (i.e. known) state of the replica. This avoids\n   * a buildup of \"work\" if a subscriber is too busy to consume all\n   * notifications.\n   */\n  subscribe(): Source<ReplicaState>;\n}\n\nexport interface Replicator extends ReplicaStateNotifier {\n  /**\n   * Returns an opaque message for human-readable consumption. This is\n   * purely for ensuring that the Replicator has started at least once to\n   * bootstrap a new replica.\n   */\n  status(): Promise<ReadonlyJSONObject>;\n}\n\nexport type ReplicatorMode = 'backup' | 'serving';\n\nexport class ReplicatorService implements Replicator, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #incrementalSyncer: IncrementalSyncer;\n  readonly #shouldPublishErrors: boolean;\n  readonly #worker: WriteWorkerClient;\n  #runPromise: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    mode: ReplicatorMode,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    statusPublisher: ReplicationStatusPublisher | null,\n  ) {\n    this.id = id;\n    this.#lc = lc\n      .withContext('component', 'replicator')\n      .withContext('serviceID', this.id);\n    this.#worker = worker;\n    this.#shouldPublishErrors = statusPublisher !== null;\n\n    this.#incrementalSyncer = new IncrementalSyncer(\n      this.#lc,\n      taskID,\n      `${taskID}/${id}`,\n      changeStreamer,\n      worker,\n      mode,\n      statusPublisher,\n    );\n  }\n\n  status() {\n    return Promise.resolve({status: 'ok'});\n  }\n\n  run() {\n    this.#runPromise = this.#incrementalSyncer.run().catch(async error => {\n      if (this.#shouldPublishErrors) {\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          'Replication stopped because the replica writer failed',\n        );\n      }\n      throw error;\n    });\n    return this.#runPromise;\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#incrementalSyncer.subscribe();\n  }\n\n  async stop() {\n    this.#incrementalSyncer.stop(this.#lc);\n    // Wait for the syncer's run loop to finish so that any in-flight\n    // worker.processMessages() call completes and clears #pending\n    // before we send the 'stop' message to the worker.\n    await this.#runPromise;\n    await this.#worker.stop();\n  }\n}\n"],"mappings":";;;AAmFA,IAAa,oBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,IACA,MACA,gBACA,QACA,iBACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GACR,YAAY,aAAa,YAAY,EACrC,YAAY,aAAa,KAAK,EAAE;EACnC,KAAKG,UAAU;EACf,KAAKD,uBAAuB,oBAAoB;EAEhD,KAAKD,qBAAqB,IAAI,kBAC5B,KAAKD,KACL,QACA,GAAG,OAAO,GAAG,MACb,gBACA,QACA,MACA,eACF;CACF;CAEA,SAAS;EACP,OAAO,QAAQ,QAAQ,EAAC,QAAQ,KAAI,CAAC;CACvC;CAEA,MAAM;EACJ,KAAKI,cAAc,KAAKH,mBAAmB,IAAI,EAAE,MAAM,OAAM,UAAS;GACpE,IAAI,KAAKC,sBACP,MAAM,wBACJ,KAAKF,KACL,eACA,uDACF;GAEF,MAAM;EACR,CAAC;EACD,OAAO,KAAKI;CACd;CAEA,YAAkC;EAChC,OAAO,KAAKH,mBAAmB,UAAU;CAC3C;CAEA,MAAM,OAAO;EACX,KAAKA,mBAAmB,KAAK,KAAKD,GAAG;EAIrC,MAAM,KAAKI;EACX,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF"}

package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts

@@ -102,14 +102,14 @@ export declare class SQLiteChangeLogObserver {
     #private;
     constructor(lc: LogContext, info: SQLiteChangeLogInfo);
     /**
-     * Called as each message arrives, before it is written. Hashes the *received*
-     * stream, which is compared against the hash the writer computes over what it
-     * *stored*.
+     * Called as each message arrives, before it is written. For a sampled
+     * transaction, hashes the *received* stream and compares it with the hash the
+     * writer computes over what it *stored*.
      */
-    messageReceived(data: ChangeStreamData, json: string): void;
+    messageReceived(data: ChangeStreamData, json: string, validateHash?: boolean, storedChange?: string): void;
     /** Called after the message has been written to the log. */
-    messageProcessed(data: ChangeStreamData, commit: ChangeLogCommit | null, durationMs: number): void;
-    messageFailed(data: ChangeStreamData, error: unknown, durationMs: number): void;
+    messageProcessed(data: ChangeStreamData, commit: ChangeLogCommit | null, durationMs: number | undefined): void;
+    messageFailed(data: ChangeStreamData, error: unknown, durationMs: number | undefined): void;
     /**
      * A committed transaction folded schema changes into the cookie jar. Reported
      * separately from {@link messageProcessed} because it costs a read of the

package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-observability.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-observability.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAQ3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AAGtF,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,eAAe,EACrB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAIL,KAAK,aAAa,EAClB,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,+BAA+B,EAAC,MAAM,+BAA+B,CAAC;AAEnF;;;;;;;;;;;GAWG;AACH,wBAAgB,8BAA8B,CAC5C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,eAAe,GACtB,IAAI,CAsCN;AAED;;;;GAIG;AACH,MAAM,MAAM,0BAA0B,GAAG,aAAa,GAAG;IACvD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,0BAA0B,GAAG;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;CACtC,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,QAAQ,GAClB,0BAA0B,CAO5B;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,QAAQ,GAClB,mBAAmB,CAgBrB;AAED,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,0BAA0B,GAC/B,IAAI,CAcN;AAED;;;;;;;;GAQG;AACH,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAcjB;AAED,gEAAgE;AAChE,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,+BAA+B,CAAC;IAChD;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,iCAAiC,GAAG;IAC9C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B,CAAC;AAIF;;;;;GAKG;AACH,qBAAa,uBAAuB;;IAuElC,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,mBAAmB,EAoDpD;IAED;;;;OAIG;IACH,eAAe,CAAC,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAsD1D;IAED,4DAA4D;IAC5D,gBAAgB,CACd,IAAI,EAAE,gBAAgB,EACtB,MAAM,EAAE,eAAe,GAAG,IAAI,EAC9B,UAAU,EAAE,MAAM,GACjB,IAAI,CA+DN;IAED,aAAa,CACX,IAAI,EAAE,gBAAgB,EACtB,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,MAAM,GACjB,IAAI,CAoBN;IAED;;;;OAIG;IACH,cAAc,CAAC,GAAG,EAAE,SAAS,WAAW,EAAE,EAAE,IAAI,EAAE,eAAe,GAAG,IAAI,CAMvE;IAED;;;;;OAKG;IACH,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAEzE;IAED,yDAAyD;IACzD,KAAK,IAAI,IAAI,CAMZ;IAED,4EAA4E;IAC5E,UAAU,CAAC,IAAI,EAAE,mBAAmB,GAAG,IAAI,CAQ1C;IAED,KAAK,IAAI,iCAAiC,CAezC;CA2DF"}
+{"version":3,"file":"sqlite-change-log-observability.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-observability.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAQ3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AAGtF,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,eAAe,EACrB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAIL,KAAK,aAAa,EAClB,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,+BAA+B,EAAC,MAAM,+BAA+B,CAAC;AAEnF;;;;;;;;;;;GAWG;AACH,wBAAgB,8BAA8B,CAC5C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,eAAe,GACtB,IAAI,CAsCN;AAED;;;;GAIG;AACH,MAAM,MAAM,0BAA0B,GAAG,aAAa,GAAG;IACvD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,0BAA0B,GAAG;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;CACtC,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,QAAQ,GAClB,0BAA0B,CAO5B;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,QAAQ,GAClB,mBAAmB,CAgBrB;AAED,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,0BAA0B,GAC/B,IAAI,CAcN;AAED;;;;;;;;GAQG;AACH,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAcjB;AAED,gEAAgE;AAChE,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,+BAA+B,CAAC;IAChD;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,iCAAiC,GAAG;IAC9C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B,CAAC;AAIF;;;;;GAKG;AACH,qBAAa,uBAAuB;;IAuElC,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,mBAAmB,EAoDpD;IAED;;;;OAIG;IACH,eAAe,CACb,IAAI,EAAE,gBAAgB,EACtB,IAAI,EAAE,MAAM,EACZ,YAAY,UAAO,EACnB,YAAY,SAA4C,GACvD,IAAI,CA4DN;IAED,4DAA4D;IAC5D,gBAAgB,CACd,IAAI,EAAE,gBAAgB,EACtB,MAAM,EAAE,eAAe,GAAG,IAAI,EAC9B,UAAU,EAAE,MAAM,GAAG,SAAS,GAC7B,IAAI,CA0EN;IAED,aAAa,CACX,IAAI,EAAE,gBAAgB,EACtB,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,MAAM,GAAG,SAAS,GAC7B,IAAI,CAsBN;IAED;;;;OAIG;IACH,cAAc,CAAC,GAAG,EAAE,SAAS,WAAW,EAAE,EAAE,IAAI,EAAE,eAAe,GAAG,IAAI,CAMvE;IAED;;;;;OAKG;IACH,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAEzE;IAED,yDAAyD;IACzD,KAAK,IAAI,IAAI,CAMZ;IAED,4EAA4E;IAC5E,UAAU,CAAC,IAAI,EAAE,mBAAmB,GAAG,IAAI,CAQ1C;IAED,KAAK,IAAI,iCAAiC,CAezC;CA+DF"}

package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js

@@ -137,7 +137,7 @@ var TRANSACTION_ROW_BUCKETS = [
 */
 var SQLiteChangeLogObserver = class {
 	#lc;
-	#messageProcessing = getOrCreateLatencyHistogram("replica", "sqlite_change_log.message_processing_duration", "Time to write a replication message to the SQLite change log.");
+	#messageProcessing = getOrCreateLatencyHistogram("replica", "sqlite_change_log.message_processing_duration", "Sampled time to write a replication message to the SQLite change log.");
 	#logCommitDuration = getOrCreateLatencyHistogram("replica", "sqlite_change_log.log_commit_duration", "Time to append the commit row to the SQLite change log and commit its transaction. This is on the forward path: it precedes the forward of that transaction's commit message.");
 	#transactionRows = getOrCreateValueHistogram("replica", "sqlite_change_log.transaction_rows", {
 		description: "Rows stored per committed SQLite change-log transaction.",
@@ -146,7 +146,7 @@ var SQLiteChangeLogObserver = class {
 	});
 	#rollbackCounter = getOrCreateCounter("replica", "sqlite_change_log.rollbacks", "SQLite change-log transactions rolled back before commit.");
 	#invariantFailureCounter = getOrCreateCounter("replica", "sqlite_change_log.invariant_failures", "Detected SQLite change-log writer invariant failures.");
-	#hashComparisonCounter = getOrCreateCounter("replica", "sqlite_change_log.hash_comparisons", "Ephemeral transaction hash comparisons between the received change stream and SQLite change-log writes.");
+	#hashComparisonCounter = getOrCreateCounter("replica", "sqlite_change_log.hash_comparisons", "Sampled ephemeral transaction hash comparisons between the received change stream and SQLite change-log writes.");
 	#hashUnpairedCounter = getOrCreateCounter("replica", "sqlite_change_log.hash_unpaired", "Committed change-log transactions without both ephemeral hashes available for comparison.");
 	#cookieMutationCounter = getOrCreateCounter("replica", "sqlite_change_log.cookie_mutations", "Change-log cookie mutations folded from schema changes, by operation.");
 	#receivedHead;
@@ -190,23 +190,23 @@ var SQLiteChangeLogObserver = class {
 		});
 	}
 	/**
-	* Called as each message arrives, before it is written. Hashes the *received*
-	* stream, which is compared against the hash the writer computes over what it
-	* *stored*.
+	* Called as each message arrives, before it is written. For a sampled
+	* transaction, hashes the *received* stream and compares it with the hash the
+	* writer computes over what it *stored*.
 	*/
-	messageReceived(data, json) {
+	messageReceived(data, json, validateHash = true, storedChange = extractChangeSubstring(json, data[1].tag)) {
 		const tag = data[1].tag;
 		if (data[0] === "begin") {
 			if (this.#sourceHasher !== void 0) this.#invariantFailure("Received change-log begin while a source hash is already open", { receivedWatermark: data[2].commitWatermark });
 			const watermark = data[2].commitWatermark;
-			this.#sourceHasher = new ChangeLogTransactionHasher();
+			this.#sourceHasher = validateHash ? new ChangeLogTransactionHasher() : void 0;
 			this.#sourcePos = 0;
 			this.#sourceCommitHash = void 0;
-			this.#sourceHasher.add({
+			this.#sourceHasher?.add({
 				watermark,
 				pos: this.#sourcePos,
 				tag,
-				change: extractChangeSubstring(json, tag),
+				change: storedChange,
 				precommit: null
 			});
 			return;
@@ -216,6 +216,7 @@ var SQLiteChangeLogObserver = class {
 			return;
 		}
 		if (data[0] === "commit") this.#receivedHead = data[2].watermark;
+		if (!validateHash) return;
 		const watermark = this.#transactionWatermark;
 		if (this.#sourceHasher === void 0 || watermark === void 0) {
 			this.#invariantFailure("Received change-log message without an observed transaction", { tag });
@@ -226,7 +227,7 @@ var SQLiteChangeLogObserver = class {
 			watermark: commitWatermark,
 			pos: ++this.#sourcePos,
 			tag,
-			change: extractChangeSubstring(json, tag),
+			change: storedChange,
 			precommit: data[0] === "commit" ? watermark : null
 		});
 		if (data[0] === "commit") {
@@ -237,7 +238,7 @@ var SQLiteChangeLogObserver = class {
 	/** Called after the message has been written to the log. */
 	messageProcessed(data, commit, durationMs) {
 		const tag = data[1].tag;
-		this.#messageProcessing.recordMs(durationMs, {
+		if (durationMs !== void 0) this.#messageProcessing.recordMs(durationMs, {
 			tag,
 			outcome: "success"
 		});
@@ -273,7 +274,7 @@ var SQLiteChangeLogObserver = class {
 			this.#rows += commit.stats.rows;
 			this.#estimatedBytes += commit.stats.estimatedBytes;
 			this.#transactionRows.record(commit.stats.rows);
-			this.#compareHashes(watermark, this.#sourceCommitHash, commit.stats.hash);
+			if (this.#sourceCommitHash !== void 0 || commit.stats.hash !== void 0) this.#compareHashes(watermark, this.#sourceCommitHash, commit.stats.hash);
 		}
 		if (watermark < this.#sqliteHead) this.#invariantFailure("SQLite change-log head regressed", {
 			previousHead: this.#sqliteHead,
@@ -285,11 +286,11 @@ var SQLiteChangeLogObserver = class {
 	}
 	messageFailed(data, error, durationMs) {
 		const tag = data[1].tag;
-		this.#messageProcessing.recordMs(durationMs, {
+		if (durationMs !== void 0) this.#messageProcessing.recordMs(durationMs, {
 			tag,
 			outcome: "error"
 		});
-		if (data[0] === "commit") this.#recordUnpaired(data[2].watermark, "sqlite-commit-failed");
+		if (data[0] === "commit" && this.#sourceCommitHash !== void 0) this.#recordUnpaired(data[2].watermark, "sqlite-commit-failed");
 		if (error instanceof Error && error.name === "ChangeLogStreamInvariantError") this.#invariantFailure(error.message);
 		if (this.#transactionWatermark !== void 0 || data[0] === "begin") this.#recordRollback("processing-error");
 		this.#transactionWatermark = void 0;
@@ -364,6 +365,10 @@ var SQLiteChangeLogObserver = class {
 			this.#recordUnpaired(watermark, "missing-source-hash");
 			return;
 		}
+		if (sqliteHash === void 0) {
+			this.#recordUnpaired(watermark, "missing-sqlite-hash");
+			return;
+		}
 		if (sourceHash === sqliteHash) {
 			this.#hashMatches++;
 			this.#hashComparisonCounter.add(1, { outcome: "match" });

package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-observability.js","names":["#lc","#messageProcessing","#logCommitDuration","#transactionRows","#rollbackCounter","#invariantFailureCounter","#hashComparisonCounter","#hashUnpairedCounter","#cookieMutationCounter","#receivedHead","#sqliteHead","#rows","#estimatedBytes","#cookieRows","#sourceHasher","#invariantFailure","#sourcePos","#sourceCommitHash","#resetSourceHash","#transactionWatermark","#recordRollback","#recordUnpaired","#compareHashes","#cookieMutations","#rollbacks","#invariantFailures","#hashMatches","#hashMismatches","#hashUnpaired"],"sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-observability.ts"],"sourcesContent":["/**\n * Observability for the SQLite change log.\n *\n * Everything here now runs in the **change-streamer** process, which is where\n * the writer lives. The module stays beside the rest of the change-log code\n * under `services/replicator/` for the same reason `change-log-db.ts` and\n * `change-log-stream-writer.ts` do: moving files would obscure the topology\n * diff, and the writer's home is a property of who calls it rather than of\n * where it sits. Metric names are unchanged from 7H — only their emitting\n * process, and therefore their resource attributes, moved.\n */\n\nimport {stat} from 'node:fs/promises';\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n  getOrCreateValueHistogram,\n} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {\n  readCookieRowCounts,\n  type CookieOpTag,\n  type CookieRowCounts,\n} from './change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  readChangeLogBounds,\n  readChangeLogMeta,\n  type ChangeLogMeta,\n  type ReconcileResult,\n} from './change-log-db.ts';\nimport type {ChangeLogStreamTransactionStats} from './change-log-stream-writer.ts';\n\n/**\n * Reports the reconciliation of the change-log database against the watermark\n * the stream connection resumes from. This runs once per stream connection, so\n * a reconnect storm is visible here as well as a restart.\n *\n * A wipe is not an error — it is how every abnormal change-log state is\n * collapsed into a correct one — but each wipe costs the retention window, so\n * every reconnecting subscriber below the head gets `too-old`. The `gap` reason\n * in particular means the log neither held the resume watermark nor could be\n * truncated to it, so a nonzero rate in steady state is an alert, not a\n * statistic.\n */\nexport function recordSQLiteChangeLogReconcile(\n  lc: LogContext,\n  result: ReconcileResult,\n): void {\n  switch (result.action) {\n    case 'truncated':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_truncated_rows',\n        'Phantom change-log rows removed at reconciliation, i.e. rows above ' +\n          'the watermark the change stream resumed from.',\n      ).add(result.rows);\n      break;\n    case 'reseeded':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_wipes',\n        'Change-log wipes at reconciliation, by reason. Each one resets the ' +\n          'retention window, so subscribers below the head get `too-old`.',\n      ).add(1, {reason: result.reason});\n      break;\n    case 'none':\n      break;\n    default:\n      unreachable(result);\n  }\n  if (result.cookiesStale) {\n    // The invariant-17 counter: after any reconcile that moved the head, the\n    // log's cookie set has to be replaced with the one that belongs to the\n    // resume watermark. A nonzero `truncated` series in steady state means\n    // phantom truncation is happening on a shard where it should not.\n    getOrCreateCounter(\n      'replica',\n      'sqlite_change_log.cookie_reseed',\n      'Change-log cookie sets invalidated at reconciliation, by the ' +\n        'reconcile action that invalidated them.',\n    ).add(1, {action: result.action});\n  }\n  lc.debug?.('SQLite change-log reconciliation', {\n    sqliteChangeLogReconcile: result,\n  });\n}\n\n/**\n * What the log says about itself. Nothing here reads the replica: after the\n * writer moved to the change-streamer, the replica's state version is a\n * consequence of this log rather than a fact about it.\n */\nexport type SQLiteChangeLogStartupInfo = ChangeLogMeta & {\n  readonly minWatermark: string;\n  readonly headWatermark: string;\n};\n\nexport type SQLiteChangeLogInfo = SQLiteChangeLogStartupInfo & {\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n};\n\n/**\n * Reads the meta row and covered range. Unlike {@link getSQLiteChangeLogInfo}\n * this does not scan the log; see {@link readChangeLogBounds}.\n *\n * Call this after reconciliation, which is the only point at which the head is\n * known to be the resume watermark.\n */\nexport function getSQLiteChangeLogStartupInfo(\n  changeLog: Database,\n): SQLiteChangeLogStartupInfo {\n  const meta = readChangeLogMeta(changeLog);\n  const {minWatermark, headWatermark} = readChangeLogBounds(changeLog);\n  if (minWatermark === null || headWatermark === null) {\n    throw new Error('the SQLite change log must contain its seed transaction');\n  }\n  return {...meta, minWatermark, headWatermark};\n}\n\n/**\n * Extends {@link getSQLiteChangeLogStartupInfo} with the retained row and\n * estimated byte totals. Computing these scans the entire change-log table, so\n * only call it when the totals are consumed, i.e. when the writer (and thus the\n * observer) is enabled.\n */\nexport function getSQLiteChangeLogInfo(\n  changeLog: Database,\n): SQLiteChangeLogInfo {\n  const startup = getSQLiteChangeLogStartupInfo(changeLog);\n  const aggregate = changeLog\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\",\n             coalesce(sum(\"estimatedBytes\"), 0) AS \"estimatedBytes\"\n        FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    `)\n    .get<{rows: number; estimatedBytes: number}>();\n\n  return {\n    ...startup,\n    rows: aggregate.rows,\n    estimatedBytes: aggregate.estimatedBytes,\n    cookieRows: readCookieRowCounts(changeLog),\n  };\n}\n\nexport function logSQLiteChangeLogStartup(\n  lc: LogContext,\n  file: string,\n  info: SQLiteChangeLogStartupInfo,\n): void {\n  lc.info?.('SQLite change-log startup', {\n    sqliteChangeLog: {\n      file,\n      epoch: info.epoch,\n      generation: info.generation,\n      replicaID: info.replicaID,\n      schemaVersion: info.schemaVersion,\n      seedWatermark: info.seedWatermark,\n      seededAtMs: info.seededAtMs,\n      minWatermark: info.minWatermark,\n      headWatermark: info.headWatermark,\n    },\n  });\n}\n\n/**\n * The on-disk footprint of a SQLite database: its main file plus its `-wal` and\n * `-wal2` sidecars, whichever exist. `-shm` is excluded — it is a\n * shared-memory index rather than retained bytes.\n *\n * A missing file counts as zero rather than warning. Which sidecars exist\n * depends on the journal mode, and the change-log database is absent entirely\n * until the writer creates it.\n */\nexport async function sqliteFileBytes(\n  lc: LogContext,\n  file: string,\n): Promise<number> {\n  const sizes = await Promise.all(\n    [file, `${file}-wal`, `${file}-wal2`].map(async path => {\n      try {\n        return (await stat(path)).size;\n      } catch (e) {\n        if ((e as NodeJS.ErrnoException).code !== 'ENOENT') {\n          lc.warn?.(`unable to stat ${path} for size metrics`, e);\n        }\n        return 0;\n      }\n    }),\n  );\n  return sizes.reduce((total, size) => total + size, 0);\n}\n\n/** What the writer reports back for a committed transaction. */\nexport type ChangeLogCommit = {\n  readonly watermark: string;\n  readonly stats: ChangeLogStreamTransactionStats;\n  /**\n   * Appending the commit row and committing the log's transaction. This is the\n   * cost that sits on the forward path, once per upstream transaction.\n   */\n  readonly logCommitMs: number;\n};\n\nexport type SQLiteChangeLogObservabilityState = {\n  readonly receivedHead: string;\n  readonly sqliteHead: string;\n  readonly headLag: number | undefined;\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n  readonly cookieMutations: number;\n  readonly rollbacks: number;\n  readonly invariantFailures: number;\n  readonly hashMatches: number;\n  readonly hashMismatches: number;\n  readonly hashUnpaired: number;\n};\n\nconst TRANSACTION_ROW_BUCKETS = [2, 3, 5, 10, 25, 50, 100, 250, 1000, 5000];\n\n/**\n * Records writer metrics in the change-streamer process. State changes only\n * after the corresponding write succeeds, except receivedHead, which advances\n * when a commit message arrives so that transient lag behind the log is\n * observable.\n */\nexport class SQLiteChangeLogObserver {\n  readonly #lc: LogContext;\n  readonly #messageProcessing = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.message_processing_duration',\n    'Time to write a replication message to the SQLite change log.',\n  );\n  // The commit that sits on the forward path. There is no second commit to\n  // order against any more: the replica's belongs to a subscriber downstream of\n  // this loop, so `sqlite_change_log.replica_commit_duration` and the phantom\n  // window it measured are gone. A failed commit is still timed, by\n  // `message_processing_duration` keyed `tag=\"commit\"`.\n  readonly #logCommitDuration = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.log_commit_duration',\n    'Time to append the commit row to the SQLite change log and commit its ' +\n      'transaction. This is on the forward path: it precedes the forward of ' +\n      \"that transaction's commit message.\",\n  );\n  readonly #transactionRows = getOrCreateValueHistogram(\n    'replica',\n    'sqlite_change_log.transaction_rows',\n    {\n      description: 'Rows stored per committed SQLite change-log transaction.',\n      unit: '{row}',\n      bucketBoundaries: TRANSACTION_ROW_BUCKETS,\n    },\n  );\n  readonly #rollbackCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.rollbacks',\n    'SQLite change-log transactions rolled back before commit.',\n  );\n  readonly #invariantFailureCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.invariant_failures',\n    'Detected SQLite change-log writer invariant failures.',\n  );\n  readonly #hashComparisonCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_comparisons',\n    'Ephemeral transaction hash comparisons between the received change stream and SQLite change-log writes.',\n  );\n  readonly #hashUnpairedCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_unpaired',\n    'Committed change-log transactions without both ephemeral hashes available for comparison.',\n  );\n  // Near zero at steady state: cookie state only moves on schema changes.\n  readonly #cookieMutationCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.cookie_mutations',\n    'Change-log cookie mutations folded from schema changes, by operation.',\n  );\n\n  #receivedHead: string;\n  #sqliteHead: string;\n  #rows: number;\n  #estimatedBytes: number;\n  #cookieRows: CookieRowCounts;\n  #cookieMutations = 0;\n  #transactionWatermark: string | undefined;\n  #sourceHasher: ChangeLogTransactionHasher | undefined;\n  #sourcePos = 0;\n  #sourceCommitHash: string | undefined;\n  #rollbacks = 0;\n  #invariantFailures = 0;\n  #hashMatches = 0;\n  #hashMismatches = 0;\n  #hashUnpaired = 0;\n\n  constructor(lc: LogContext, info: SQLiteChangeLogInfo) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-observer');\n    // Reconciliation has just put the head at the resume watermark, which is\n    // also the point the stream is about to be received from.\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.rows',\n      'Rows retained in the SQLite change log.',\n    ).addCallback(result => result.observe(this.#rows));\n    // Nonzero with no backfill in flight is a leaked cookie, i.e. a backfill\n    // that will be re-requested forever.\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.cookie_rows',\n      'Cookie rows retained in the SQLite change log, by cookie table.',\n    ).addCallback(result => {\n      result.observe(this.#cookieRows.tableMetadata, {table: 'tableMetadata'});\n      result.observe(this.#cookieRows.backfilling, {table: 'backfilling'});\n    });\n    getOrCreateGauge('replica', 'sqlite_change_log.retained_bytes', {\n      description:\n        'Estimated UTF-8 payload bytes retained in the SQLite change log.',\n      unit: 'By',\n    }).addCallback(result => result.observe(this.#estimatedBytes));\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head',\n      'SQLite change-log head converted from its lexicographic watermark.',\n    ).addCallback(result => {\n      const head = watermarkValue(this.#sqliteHead);\n      if (head !== undefined) {\n        result.observe(head);\n      }\n    });\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head_lag',\n      'Distance from the latest received commit to the SQLite change-log ' +\n        'head. The log commits before the forward, so this is zero except ' +\n        'within a transaction.',\n    ).addCallback(result => {\n      const lag = watermarkDistance(this.#receivedHead, this.#sqliteHead);\n      if (lag !== undefined) {\n        result.observe(lag);\n      }\n    });\n  }\n\n  /**\n   * Called as each message arrives, before it is written. Hashes the *received*\n   * stream, which is compared against the hash the writer computes over what it\n   * *stored*.\n   */\n  messageReceived(data: ChangeStreamData, json: string): void {\n    const tag = data[1].tag;\n    if (data[0] === 'begin') {\n      if (this.#sourceHasher !== undefined) {\n        this.#invariantFailure(\n          'Received change-log begin while a source hash is already open',\n          {receivedWatermark: data[2].commitWatermark},\n        );\n      }\n      const watermark = data[2].commitWatermark;\n      this.#sourceHasher = new ChangeLogTransactionHasher();\n      this.#sourcePos = 0;\n      this.#sourceCommitHash = undefined;\n      this.#sourceHasher.add({\n        watermark,\n        pos: this.#sourcePos,\n        tag,\n        change: extractChangeSubstring(json, tag),\n        precommit: null,\n      });\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] === 'commit') {\n      this.#receivedHead = data[2].watermark;\n    }\n\n    const watermark = this.#transactionWatermark;\n    if (this.#sourceHasher === undefined || watermark === undefined) {\n      this.#invariantFailure(\n        'Received change-log message without an observed transaction',\n        {tag},\n      );\n      return;\n    }\n\n    const commitWatermark =\n      data[0] === 'commit' ? data[2].watermark : watermark;\n    this.#sourceHasher.add({\n      watermark: commitWatermark,\n      pos: ++this.#sourcePos,\n      tag,\n      change: extractChangeSubstring(json, tag),\n      precommit: data[0] === 'commit' ? watermark : null,\n    });\n    if (data[0] === 'commit') {\n      this.#sourceCommitHash = this.#sourceHasher.digest();\n      this.#sourceHasher = undefined;\n    }\n  }\n\n  /** Called after the message has been written to the log. */\n  messageProcessed(\n    data: ChangeStreamData,\n    commit: ChangeLogCommit | null,\n    durationMs: number,\n  ): void {\n    const tag = data[1].tag;\n    this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'success'});\n\n    if (data[0] === 'begin') {\n      if (this.#transactionWatermark !== undefined) {\n        this.#invariantFailure('SQLite change-log transaction already open', {\n          openWatermark: this.#transactionWatermark,\n          receivedWatermark: data[2].commitWatermark,\n        });\n      }\n      this.#transactionWatermark = data[2].commitWatermark;\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#recordRollback('upstream');\n      this.#transactionWatermark = undefined;\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] !== 'commit') {\n      return;\n    }\n\n    const watermark = data[2].watermark;\n    if (commit !== null) {\n      this.#logCommitDuration.recordMs(commit.logCommitMs);\n    }\n    if (this.#transactionWatermark !== watermark) {\n      this.#invariantFailure(\n        'SQLite change-log commit does not match the observed begin',\n        {openWatermark: this.#transactionWatermark, commitWatermark: watermark},\n      );\n    }\n    if (commit === null) {\n      this.#invariantFailure(\n        'SQLite change-log commit result is missing writer statistics',\n        {commitWatermark: watermark},\n      );\n      this.#recordUnpaired(watermark, 'missing-sqlite-hash');\n    } else {\n      if (commit.watermark !== watermark) {\n        this.#invariantFailure(\n          'SQLite change-log commit result has an unexpected watermark',\n          {commitWatermark: watermark, resultWatermark: commit.watermark},\n        );\n      }\n      this.#rows += commit.stats.rows;\n      this.#estimatedBytes += commit.stats.estimatedBytes;\n      this.#transactionRows.record(commit.stats.rows);\n      this.#compareHashes(watermark, this.#sourceCommitHash, commit.stats.hash);\n    }\n    if (watermark < this.#sqliteHead) {\n      this.#invariantFailure('SQLite change-log head regressed', {\n        previousHead: this.#sqliteHead,\n        commitWatermark: watermark,\n      });\n    }\n    this.#sqliteHead = watermark;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  messageFailed(\n    data: ChangeStreamData,\n    error: unknown,\n    durationMs: number,\n  ): void {\n    const tag = data[1].tag;\n    this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'error'});\n    if (data[0] === 'commit') {\n      // No commit duration here: a failed commit produces no statistics, so\n      // there is nothing to attribute. The attempt is timed by\n      // `message_processing_duration{tag=\"commit\",outcome=\"error\"}` above.\n      this.#recordUnpaired(data[2].watermark, 'sqlite-commit-failed');\n    }\n    if (\n      error instanceof Error &&\n      error.name === 'ChangeLogStreamInvariantError'\n    ) {\n      this.#invariantFailure(error.message);\n    }\n    if (this.#transactionWatermark !== undefined || data[0] === 'begin') {\n      this.#recordRollback('processing-error');\n    }\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  /**\n   * A committed transaction folded schema changes into the cookie jar. Reported\n   * separately from {@link messageProcessed} because it costs a read of the\n   * cookie tables, which is only worth taking when they actually moved.\n   */\n  cookiesMutated(ops: readonly CookieOpTag[], rows: CookieRowCounts): void {\n    for (const op of ops) {\n      this.#cookieMutations++;\n      this.#cookieMutationCounter.add(1, {op});\n    }\n    this.#cookieRows = rows;\n  }\n\n  /**\n   * A missed truncate-above surfaces as a constraint violation on the writer's\n   * plain `INSERT`, which the fail-soft policy would otherwise absorb quietly.\n   * Classifying it as an invariant failure is what keeps a reconciliation bug\n   * from looking like a lost cache.\n   */\n  invariantFailure(message: string, details?: Record<string, unknown>): void {\n    this.#invariantFailure(message, details);\n  }\n\n  /** The source interrupted the stream mid-transaction. */\n  abort(): void {\n    if (this.#transactionWatermark !== undefined) {\n      this.#recordRollback('source-interruption');\n      this.#transactionWatermark = undefined;\n    }\n    this.#resetSourceHash();\n  }\n\n  /** Reconciliation moved the head, so the observer's totals are restated. */\n  reconciled(info: SQLiteChangeLogInfo): void {\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  state(): SQLiteChangeLogObservabilityState {\n    return {\n      receivedHead: this.#receivedHead,\n      sqliteHead: this.#sqliteHead,\n      headLag: watermarkDistance(this.#receivedHead, this.#sqliteHead),\n      rows: this.#rows,\n      estimatedBytes: this.#estimatedBytes,\n      cookieRows: this.#cookieRows,\n      cookieMutations: this.#cookieMutations,\n      rollbacks: this.#rollbacks,\n      invariantFailures: this.#invariantFailures,\n      hashMatches: this.#hashMatches,\n      hashMismatches: this.#hashMismatches,\n      hashUnpaired: this.#hashUnpaired,\n    };\n  }\n\n  #recordRollback(\n    reason: 'upstream' | 'processing-error' | 'source-interruption',\n  ) {\n    this.#rollbacks++;\n    this.#rollbackCounter.add(1, {reason});\n  }\n\n  #invariantFailure(message: string, details?: Record<string, unknown>) {\n    this.#invariantFailures++;\n    this.#invariantFailureCounter.add(1);\n    this.#lc.error?.(message, details);\n  }\n\n  #compareHashes(\n    watermark: string,\n    sourceHash: string | undefined,\n    sqliteHash: string,\n  ): void {\n    if (sourceHash === undefined) {\n      this.#recordUnpaired(watermark, 'missing-source-hash');\n      return;\n    }\n    if (sourceHash === sqliteHash) {\n      this.#hashMatches++;\n      this.#hashComparisonCounter.add(1, {outcome: 'match'});\n      return;\n    }\n    this.#hashMismatches++;\n    this.#hashComparisonCounter.add(1, {outcome: 'mismatch'});\n    this.#lc.error?.('SQLite change-log transaction hash mismatch', {\n      sqliteChangeLogHashComparison: {\n        watermark,\n        sourceHash: sourceHash.slice(0, 16),\n        sqliteHash: sqliteHash.slice(0, 16),\n      },\n    });\n  }\n\n  #recordUnpaired(\n    watermark: string,\n    reason:\n      | 'missing-source-hash'\n      | 'missing-sqlite-hash'\n      | 'sqlite-commit-failed',\n  ): void {\n    this.#hashUnpaired++;\n    this.#hashUnpairedCounter.add(1, {reason});\n    this.#lc.warn?.('SQLite change-log transaction hash was not compared', {\n      sqliteChangeLogHashComparison: {watermark, reason},\n    });\n  }\n\n  #resetSourceHash(): void {\n    this.#sourceHasher = undefined;\n    this.#sourcePos = 0;\n    this.#sourceCommitHash = undefined;\n  }\n}\n\nfunction watermarkValue(watermark: string): number | undefined {\n  try {\n    const value = Number(versionFromLexi(watermark));\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n\nfunction watermarkDistance(\n  receivedHead: string,\n  sqliteHead: string,\n): number | undefined {\n  if (receivedHead <= sqliteHead) {\n    return 0;\n  }\n  try {\n    const distance =\n      versionFromLexi(receivedHead) - versionFromLexi(sqliteHead);\n    const value = Number(distance);\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA,SAAgB,+BACd,IACA,QACM;CACN,QAAQ,OAAO,QAAf;EACE,KAAK;GACH,mBACE,WACA,8CACA,kHAEF,EAAE,IAAI,OAAO,IAAI;GACjB;EACF,KAAK;GACH,mBACE,WACA,qCACA,mIAEF,EAAE,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;GAChC;EACF,KAAK,QACH;EACF,SACE,YAAY,MAAM;CACtB;CACA,IAAI,OAAO,cAKT,mBACE,WACA,mCACA,sGAEF,EAAE,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;CAElC,GAAG,QAAQ,oCAAoC,EAC7C,0BAA0B,OAC5B,CAAC;AACH;;;;;;;;AAyBA,SAAgB,8BACd,WAC4B;CAC5B,MAAM,OAAO,kBAAkB,SAAS;CACxC,MAAM,EAAC,cAAc,kBAAiB,oBAAoB,SAAS;CACnE,IAAI,iBAAiB,QAAQ,kBAAkB,MAC7C,MAAM,IAAI,MAAM,yDAAyD;CAE3E,OAAO;EAAC,GAAG;EAAM;EAAc;CAAa;AAC9C;;;;;;;AAQA,SAAgB,uBACd,WACqB;CACrB,MAAM,UAAU,8BAA8B,SAAS;CACvD,MAAM,YAAY,UACf,QAAgB;;;gBAGL,wBAAwB;KACnC,EACA,IAA4C;CAE/C,OAAO;EACL,GAAG;EACH,MAAM,UAAU;EAChB,gBAAgB,UAAU;EAC1B,YAAY,oBAAoB,SAAS;CAC3C;AACF;AAEA,SAAgB,0BACd,IACA,MACA,MACM;CACN,GAAG,OAAO,6BAA6B,EACrC,iBAAiB;EACf;EACA,OAAO,KAAK;EACZ,YAAY,KAAK;EACjB,WAAW,KAAK;EAChB,eAAe,KAAK;EACpB,eAAe,KAAK;EACpB,YAAY,KAAK;EACjB,cAAc,KAAK;EACnB,eAAe,KAAK;CACtB,EACF,CAAC;AACH;;;;;;;;;;AAWA,eAAsB,gBACpB,IACA,MACiB;CAajB,QAAO,MAZa,QAAQ,IAC1B;EAAC;EAAM,GAAG,KAAK;EAAO,GAAG,KAAK;CAAM,EAAE,IAAI,OAAM,SAAQ;EACtD,IAAI;GACF,QAAQ,MAAM,KAAK,IAAI,GAAG;EAC5B,SAAS,GAAG;GACV,IAAK,EAA4B,SAAS,UACxC,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;GAExD,OAAO;EACT;CACF,CAAC,CACH,GACa,QAAQ,OAAO,SAAS,QAAQ,MAAM,CAAC;AACtD;AA4BA,IAAM,0BAA0B;CAAC;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAM;AAAI;;;;;;;AAQ1E,IAAa,0BAAb,MAAqC;CACnC;CACA,qBAA8B,4BAC5B,WACA,iDACA,+DACF;CAMA,qBAA8B,4BAC5B,WACA,yCACA,+KAGF;CACA,mBAA4B,0BAC1B,WACA,sCACA;EACE,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CACF;CACA,mBAA4B,mBAC1B,WACA,+BACA,2DACF;CACA,2BAAoC,mBAClC,WACA,wCACA,uDACF;CACA,yBAAkC,mBAChC,WACA,sCACA,yGACF;CACA,uBAAgC,mBAC9B,WACA,mCACA,2FACF;CAEA,yBAAkC,mBAChC,WACA,sCACA,uEACF;CAEA;CACA;CACA;CACA;CACA;CACA,mBAAmB;CACnB;CACA;CACA,aAAa;CACb;CACA,aAAa;CACb,qBAAqB;CACrB,eAAe;CACf,kBAAkB;CAClB,gBAAgB;CAEhB,YAAY,IAAgB,MAA2B;EACrD,KAAKA,MAAM,GAAG,YAAY,aAAa,4BAA4B;EAGnE,KAAKS,gBAAgB,KAAK;EAC1B,KAAKC,cAAc,KAAK;EACxB,KAAKC,QAAQ,KAAK;EAClB,KAAKC,kBAAkB,KAAK;EAC5B,KAAKC,cAAc,KAAK;EAExB,iBACE,WACA,0BACA,yCACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKF,KAAK,CAAC;EAGlD,iBACE,WACA,iCACA,iEACF,EAAE,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAKE,YAAY,eAAe,EAAC,OAAO,gBAAe,CAAC;GACvE,OAAO,QAAQ,KAAKA,YAAY,aAAa,EAAC,OAAO,cAAa,CAAC;EACrE,CAAC;EACD,iBAAiB,WAAW,oCAAoC;GAC9D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKD,eAAe,CAAC;EAC7D,iBACE,WACA,0BACA,oEACF,EAAE,aAAY,WAAU;GACtB,MAAM,OAAO,eAAe,KAAKF,WAAW;GAC5C,IAAI,SAAS,KAAA,GACX,OAAO,QAAQ,IAAI;EAEvB,CAAC;EACD,iBACE,WACA,8BACA,0JAGF,EAAE,aAAY,WAAU;GACtB,MAAM,MAAM,kBAAkB,KAAKD,eAAe,KAAKC,WAAW;GAClE,IAAI,QAAQ,KAAA,GACV,OAAO,QAAQ,GAAG;EAEtB,CAAC;CACH;;;;;;CAOA,gBAAgB,MAAwB,MAAoB;EAC1D,MAAM,MAAM,KAAK,GAAG;EACpB,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAKI,kBAAkB,KAAA,GACzB,KAAKC,kBACH,iEACA,EAAC,mBAAmB,KAAK,GAAG,gBAAe,CAC7C;GAEF,MAAM,YAAY,KAAK,GAAG;GAC1B,KAAKD,gBAAgB,IAAI,2BAA2B;GACpD,KAAKE,aAAa;GAClB,KAAKC,oBAAoB,KAAA;GACzB,KAAKH,cAAc,IAAI;IACrB;IACA,KAAK,KAAKE;IACV;IACA,QAAQ,uBAAuB,MAAM,GAAG;IACxC,WAAW;GACb,CAAC;GACD;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAKE,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd,KAAKT,gBAAgB,KAAK,GAAG;EAG/B,MAAM,YAAY,KAAKU;EACvB,IAAI,KAAKL,kBAAkB,KAAA,KAAa,cAAc,KAAA,GAAW;GAC/D,KAAKC,kBACH,+DACA,EAAC,IAAG,CACN;GACA;EACF;EAEA,MAAM,kBACJ,KAAK,OAAO,WAAW,KAAK,GAAG,YAAY;EAC7C,KAAKD,cAAc,IAAI;GACrB,WAAW;GACX,KAAK,EAAE,KAAKE;GACZ;GACA,QAAQ,uBAAuB,MAAM,GAAG;GACxC,WAAW,KAAK,OAAO,WAAW,YAAY;EAChD,CAAC;EACD,IAAI,KAAK,OAAO,UAAU;GACxB,KAAKC,oBAAoB,KAAKH,cAAc,OAAO;GACnD,KAAKA,gBAAgB,KAAA;EACvB;CACF;;CAGA,iBACE,MACA,QACA,YACM;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,KAAKb,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAS,CAAC;EAEtE,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAKkB,0BAA0B,KAAA,GACjC,KAAKJ,kBAAkB,8CAA8C;IACnE,eAAe,KAAKI;IACpB,mBAAmB,KAAK,GAAG;GAC7B,CAAC;GAEH,KAAKA,wBAAwB,KAAK,GAAG;GACrC;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAKC,gBAAgB,UAAU;GAC/B,KAAKD,wBAAwB,KAAA;GAC7B,KAAKD,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd;EAGF,MAAM,YAAY,KAAK,GAAG;EAC1B,IAAI,WAAW,MACb,KAAKhB,mBAAmB,SAAS,OAAO,WAAW;EAErD,IAAI,KAAKiB,0BAA0B,WACjC,KAAKJ,kBACH,8DACA;GAAC,eAAe,KAAKI;GAAuB,iBAAiB;EAAS,CACxE;EAEF,IAAI,WAAW,MAAM;GACnB,KAAKJ,kBACH,gEACA,EAAC,iBAAiB,UAAS,CAC7B;GACA,KAAKM,gBAAgB,WAAW,qBAAqB;EACvD,OAAO;GACL,IAAI,OAAO,cAAc,WACvB,KAAKN,kBACH,+DACA;IAAC,iBAAiB;IAAW,iBAAiB,OAAO;GAAS,CAChE;GAEF,KAAKJ,SAAS,OAAO,MAAM;GAC3B,KAAKC,mBAAmB,OAAO,MAAM;GACrC,KAAKT,iBAAiB,OAAO,OAAO,MAAM,IAAI;GAC9C,KAAKmB,eAAe,WAAW,KAAKL,mBAAmB,OAAO,MAAM,IAAI;EAC1E;EACA,IAAI,YAAY,KAAKP,aACnB,KAAKK,kBAAkB,oCAAoC;GACzD,cAAc,KAAKL;GACnB,iBAAiB;EACnB,CAAC;EAEH,KAAKA,cAAc;EACnB,KAAKS,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;CAEA,cACE,MACA,OACA,YACM;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,KAAKjB,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAO,CAAC;EACpE,IAAI,KAAK,OAAO,UAId,KAAKoB,gBAAgB,KAAK,GAAG,WAAW,sBAAsB;EAEhE,IACE,iBAAiB,SACjB,MAAM,SAAS,iCAEf,KAAKN,kBAAkB,MAAM,OAAO;EAEtC,IAAI,KAAKI,0BAA0B,KAAA,KAAa,KAAK,OAAO,SAC1D,KAAKC,gBAAgB,kBAAkB;EAEzC,KAAKD,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;;;;;;CAOA,eAAe,KAA6B,MAA6B;EACvE,KAAK,MAAM,MAAM,KAAK;GACpB,KAAKK;GACL,KAAKf,uBAAuB,IAAI,GAAG,EAAC,GAAE,CAAC;EACzC;EACA,KAAKK,cAAc;CACrB;;;;;;;CAQA,iBAAiB,SAAiB,SAAyC;EACzE,KAAKE,kBAAkB,SAAS,OAAO;CACzC;;CAGA,QAAc;EACZ,IAAI,KAAKI,0BAA0B,KAAA,GAAW;GAC5C,KAAKC,gBAAgB,qBAAqB;GAC1C,KAAKD,wBAAwB,KAAA;EAC/B;EACA,KAAKD,iBAAiB;CACxB;;CAGA,WAAW,MAAiC;EAC1C,KAAKT,gBAAgB,KAAK;EAC1B,KAAKC,cAAc,KAAK;EACxB,KAAKC,QAAQ,KAAK;EAClB,KAAKC,kBAAkB,KAAK;EAC5B,KAAKC,cAAc,KAAK;EACxB,KAAKM,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;CAEA,QAA2C;EACzC,OAAO;GACL,cAAc,KAAKT;GACnB,YAAY,KAAKC;GACjB,SAAS,kBAAkB,KAAKD,eAAe,KAAKC,WAAW;GAC/D,MAAM,KAAKC;GACX,gBAAgB,KAAKC;GACrB,YAAY,KAAKC;GACjB,iBAAiB,KAAKU;GACtB,WAAW,KAAKC;GAChB,mBAAmB,KAAKC;GACxB,aAAa,KAAKC;GAClB,gBAAgB,KAAKC;GACrB,cAAc,KAAKC;EACrB;CACF;CAEA,gBACE,QACA;EACA,KAAKJ;EACL,KAAKpB,iBAAiB,IAAI,GAAG,EAAC,OAAM,CAAC;CACvC;CAEA,kBAAkB,SAAiB,SAAmC;EACpE,KAAKqB;EACL,KAAKpB,yBAAyB,IAAI,CAAC;EACnC,KAAKL,IAAI,QAAQ,SAAS,OAAO;CACnC;CAEA,eACE,WACA,YACA,YACM;EACN,IAAI,eAAe,KAAA,GAAW;GAC5B,KAAKqB,gBAAgB,WAAW,qBAAqB;GACrD;EACF;EACA,IAAI,eAAe,YAAY;GAC7B,KAAKK;GACL,KAAKpB,uBAAuB,IAAI,GAAG,EAAC,SAAS,QAAO,CAAC;GACrD;EACF;EACA,KAAKqB;EACL,KAAKrB,uBAAuB,IAAI,GAAG,EAAC,SAAS,WAAU,CAAC;EACxD,KAAKN,IAAI,QAAQ,+CAA+C,EAC9D,+BAA+B;GAC7B;GACA,YAAY,WAAW,MAAM,GAAG,EAAE;GAClC,YAAY,WAAW,MAAM,GAAG,EAAE;EACpC,EACF,CAAC;CACH;CAEA,gBACE,WACA,QAIM;EACN,KAAK4B;EACL,KAAKrB,qBAAqB,IAAI,GAAG,EAAC,OAAM,CAAC;EACzC,KAAKP,IAAI,OAAO,uDAAuD,EACrE,+BAA+B;GAAC;GAAW;EAAM,EACnD,CAAC;CACH;CAEA,mBAAyB;EACvB,KAAKc,gBAAgB,KAAA;EACrB,KAAKE,aAAa;EAClB,KAAKC,oBAAoB,KAAA;CAC3B;AACF;AAEA,SAAS,eAAe,WAAuC;CAC7D,IAAI;EACF,MAAM,QAAQ,OAAO,gBAAgB,SAAS,CAAC;EAC/C,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF;AAEA,SAAS,kBACP,cACA,YACoB;CACpB,IAAI,gBAAgB,YAClB,OAAO;CAET,IAAI;EACF,MAAM,WACJ,gBAAgB,YAAY,IAAI,gBAAgB,UAAU;EAC5D,MAAM,QAAQ,OAAO,QAAQ;EAC7B,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF"}
+{"version":3,"file":"sqlite-change-log-observability.js","names":["#lc","#messageProcessing","#logCommitDuration","#transactionRows","#rollbackCounter","#invariantFailureCounter","#hashComparisonCounter","#hashUnpairedCounter","#cookieMutationCounter","#receivedHead","#sqliteHead","#rows","#estimatedBytes","#cookieRows","#sourceHasher","#invariantFailure","#sourcePos","#sourceCommitHash","#resetSourceHash","#transactionWatermark","#recordRollback","#recordUnpaired","#compareHashes","#cookieMutations","#rollbacks","#invariantFailures","#hashMatches","#hashMismatches","#hashUnpaired"],"sources":["../../../../../../zero-cache/src/services/replicator/sqlite-change-log-observability.ts"],"sourcesContent":["/**\n * Observability for the SQLite change log.\n *\n * Everything here now runs in the **change-streamer** process, which is where\n * the writer lives. The module stays beside the rest of the change-log code\n * under `services/replicator/` for the same reason `change-log-db.ts` and\n * `change-log-stream-writer.ts` do: moving files would obscure the topology\n * diff, and the writer's home is a property of who calls it rather than of\n * where it sits. Metric names are unchanged from 7H — only their emitting\n * process, and therefore their resource attributes, moved.\n */\n\nimport {stat} from 'node:fs/promises';\nimport type {LogContext} from '@rocicorp/logger';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n  getOrCreateValueHistogram,\n} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {\n  readCookieRowCounts,\n  type CookieOpTag,\n  type CookieRowCounts,\n} from './change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  readChangeLogBounds,\n  readChangeLogMeta,\n  type ChangeLogMeta,\n  type ReconcileResult,\n} from './change-log-db.ts';\nimport type {ChangeLogStreamTransactionStats} from './change-log-stream-writer.ts';\n\n/**\n * Reports the reconciliation of the change-log database against the watermark\n * the stream connection resumes from. This runs once per stream connection, so\n * a reconnect storm is visible here as well as a restart.\n *\n * A wipe is not an error — it is how every abnormal change-log state is\n * collapsed into a correct one — but each wipe costs the retention window, so\n * every reconnecting subscriber below the head gets `too-old`. The `gap` reason\n * in particular means the log neither held the resume watermark nor could be\n * truncated to it, so a nonzero rate in steady state is an alert, not a\n * statistic.\n */\nexport function recordSQLiteChangeLogReconcile(\n  lc: LogContext,\n  result: ReconcileResult,\n): void {\n  switch (result.action) {\n    case 'truncated':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_truncated_rows',\n        'Phantom change-log rows removed at reconciliation, i.e. rows above ' +\n          'the watermark the change stream resumed from.',\n      ).add(result.rows);\n      break;\n    case 'reseeded':\n      getOrCreateCounter(\n        'replica',\n        'sqlite_change_log.reconcile_wipes',\n        'Change-log wipes at reconciliation, by reason. Each one resets the ' +\n          'retention window, so subscribers below the head get `too-old`.',\n      ).add(1, {reason: result.reason});\n      break;\n    case 'none':\n      break;\n    default:\n      unreachable(result);\n  }\n  if (result.cookiesStale) {\n    // The invariant-17 counter: after any reconcile that moved the head, the\n    // log's cookie set has to be replaced with the one that belongs to the\n    // resume watermark. A nonzero `truncated` series in steady state means\n    // phantom truncation is happening on a shard where it should not.\n    getOrCreateCounter(\n      'replica',\n      'sqlite_change_log.cookie_reseed',\n      'Change-log cookie sets invalidated at reconciliation, by the ' +\n        'reconcile action that invalidated them.',\n    ).add(1, {action: result.action});\n  }\n  lc.debug?.('SQLite change-log reconciliation', {\n    sqliteChangeLogReconcile: result,\n  });\n}\n\n/**\n * What the log says about itself. Nothing here reads the replica: after the\n * writer moved to the change-streamer, the replica's state version is a\n * consequence of this log rather than a fact about it.\n */\nexport type SQLiteChangeLogStartupInfo = ChangeLogMeta & {\n  readonly minWatermark: string;\n  readonly headWatermark: string;\n};\n\nexport type SQLiteChangeLogInfo = SQLiteChangeLogStartupInfo & {\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n};\n\n/**\n * Reads the meta row and covered range. Unlike {@link getSQLiteChangeLogInfo}\n * this does not scan the log; see {@link readChangeLogBounds}.\n *\n * Call this after reconciliation, which is the only point at which the head is\n * known to be the resume watermark.\n */\nexport function getSQLiteChangeLogStartupInfo(\n  changeLog: Database,\n): SQLiteChangeLogStartupInfo {\n  const meta = readChangeLogMeta(changeLog);\n  const {minWatermark, headWatermark} = readChangeLogBounds(changeLog);\n  if (minWatermark === null || headWatermark === null) {\n    throw new Error('the SQLite change log must contain its seed transaction');\n  }\n  return {...meta, minWatermark, headWatermark};\n}\n\n/**\n * Extends {@link getSQLiteChangeLogStartupInfo} with the retained row and\n * estimated byte totals. Computing these scans the entire change-log table, so\n * only call it when the totals are consumed, i.e. when the writer (and thus the\n * observer) is enabled.\n */\nexport function getSQLiteChangeLogInfo(\n  changeLog: Database,\n): SQLiteChangeLogInfo {\n  const startup = getSQLiteChangeLogStartupInfo(changeLog);\n  const aggregate = changeLog\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\",\n             coalesce(sum(\"estimatedBytes\"), 0) AS \"estimatedBytes\"\n        FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n    `)\n    .get<{rows: number; estimatedBytes: number}>();\n\n  return {\n    ...startup,\n    rows: aggregate.rows,\n    estimatedBytes: aggregate.estimatedBytes,\n    cookieRows: readCookieRowCounts(changeLog),\n  };\n}\n\nexport function logSQLiteChangeLogStartup(\n  lc: LogContext,\n  file: string,\n  info: SQLiteChangeLogStartupInfo,\n): void {\n  lc.info?.('SQLite change-log startup', {\n    sqliteChangeLog: {\n      file,\n      epoch: info.epoch,\n      generation: info.generation,\n      replicaID: info.replicaID,\n      schemaVersion: info.schemaVersion,\n      seedWatermark: info.seedWatermark,\n      seededAtMs: info.seededAtMs,\n      minWatermark: info.minWatermark,\n      headWatermark: info.headWatermark,\n    },\n  });\n}\n\n/**\n * The on-disk footprint of a SQLite database: its main file plus its `-wal` and\n * `-wal2` sidecars, whichever exist. `-shm` is excluded — it is a\n * shared-memory index rather than retained bytes.\n *\n * A missing file counts as zero rather than warning. Which sidecars exist\n * depends on the journal mode, and the change-log database is absent entirely\n * until the writer creates it.\n */\nexport async function sqliteFileBytes(\n  lc: LogContext,\n  file: string,\n): Promise<number> {\n  const sizes = await Promise.all(\n    [file, `${file}-wal`, `${file}-wal2`].map(async path => {\n      try {\n        return (await stat(path)).size;\n      } catch (e) {\n        if ((e as NodeJS.ErrnoException).code !== 'ENOENT') {\n          lc.warn?.(`unable to stat ${path} for size metrics`, e);\n        }\n        return 0;\n      }\n    }),\n  );\n  return sizes.reduce((total, size) => total + size, 0);\n}\n\n/** What the writer reports back for a committed transaction. */\nexport type ChangeLogCommit = {\n  readonly watermark: string;\n  readonly stats: ChangeLogStreamTransactionStats;\n  /**\n   * Appending the commit row and committing the log's transaction. This is the\n   * cost that sits on the forward path, once per upstream transaction.\n   */\n  readonly logCommitMs: number;\n};\n\nexport type SQLiteChangeLogObservabilityState = {\n  readonly receivedHead: string;\n  readonly sqliteHead: string;\n  readonly headLag: number | undefined;\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly cookieRows: CookieRowCounts;\n  readonly cookieMutations: number;\n  readonly rollbacks: number;\n  readonly invariantFailures: number;\n  readonly hashMatches: number;\n  readonly hashMismatches: number;\n  readonly hashUnpaired: number;\n};\n\nconst TRANSACTION_ROW_BUCKETS = [2, 3, 5, 10, 25, 50, 100, 250, 1000, 5000];\n\n/**\n * Records writer metrics in the change-streamer process. State changes only\n * after the corresponding write succeeds, except receivedHead, which advances\n * when a commit message arrives so that transient lag behind the log is\n * observable.\n */\nexport class SQLiteChangeLogObserver {\n  readonly #lc: LogContext;\n  readonly #messageProcessing = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.message_processing_duration',\n    'Sampled time to write a replication message to the SQLite change log.',\n  );\n  // The commit that sits on the forward path. There is no second commit to\n  // order against any more: the replica's belongs to a subscriber downstream of\n  // this loop, so `sqlite_change_log.replica_commit_duration` and the phantom\n  // window it measured are gone. Sampled failed commits are still timed by\n  // `message_processing_duration` keyed `tag=\"commit\"`.\n  readonly #logCommitDuration = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.log_commit_duration',\n    'Time to append the commit row to the SQLite change log and commit its ' +\n      'transaction. This is on the forward path: it precedes the forward of ' +\n      \"that transaction's commit message.\",\n  );\n  readonly #transactionRows = getOrCreateValueHistogram(\n    'replica',\n    'sqlite_change_log.transaction_rows',\n    {\n      description: 'Rows stored per committed SQLite change-log transaction.',\n      unit: '{row}',\n      bucketBoundaries: TRANSACTION_ROW_BUCKETS,\n    },\n  );\n  readonly #rollbackCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.rollbacks',\n    'SQLite change-log transactions rolled back before commit.',\n  );\n  readonly #invariantFailureCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.invariant_failures',\n    'Detected SQLite change-log writer invariant failures.',\n  );\n  readonly #hashComparisonCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_comparisons',\n    'Sampled ephemeral transaction hash comparisons between the received change stream and SQLite change-log writes.',\n  );\n  readonly #hashUnpairedCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.hash_unpaired',\n    'Committed change-log transactions without both ephemeral hashes available for comparison.',\n  );\n  // Near zero at steady state: cookie state only moves on schema changes.\n  readonly #cookieMutationCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.cookie_mutations',\n    'Change-log cookie mutations folded from schema changes, by operation.',\n  );\n\n  #receivedHead: string;\n  #sqliteHead: string;\n  #rows: number;\n  #estimatedBytes: number;\n  #cookieRows: CookieRowCounts;\n  #cookieMutations = 0;\n  #transactionWatermark: string | undefined;\n  #sourceHasher: ChangeLogTransactionHasher | undefined;\n  #sourcePos = 0;\n  #sourceCommitHash: string | undefined;\n  #rollbacks = 0;\n  #invariantFailures = 0;\n  #hashMatches = 0;\n  #hashMismatches = 0;\n  #hashUnpaired = 0;\n\n  constructor(lc: LogContext, info: SQLiteChangeLogInfo) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-observer');\n    // Reconciliation has just put the head at the resume watermark, which is\n    // also the point the stream is about to be received from.\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.rows',\n      'Rows retained in the SQLite change log.',\n    ).addCallback(result => result.observe(this.#rows));\n    // Nonzero with no backfill in flight is a leaked cookie, i.e. a backfill\n    // that will be re-requested forever.\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.cookie_rows',\n      'Cookie rows retained in the SQLite change log, by cookie table.',\n    ).addCallback(result => {\n      result.observe(this.#cookieRows.tableMetadata, {table: 'tableMetadata'});\n      result.observe(this.#cookieRows.backfilling, {table: 'backfilling'});\n    });\n    getOrCreateGauge('replica', 'sqlite_change_log.retained_bytes', {\n      description:\n        'Estimated UTF-8 payload bytes retained in the SQLite change log.',\n      unit: 'By',\n    }).addCallback(result => result.observe(this.#estimatedBytes));\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head',\n      'SQLite change-log head converted from its lexicographic watermark.',\n    ).addCallback(result => {\n      const head = watermarkValue(this.#sqliteHead);\n      if (head !== undefined) {\n        result.observe(head);\n      }\n    });\n    getOrCreateGauge(\n      'replica',\n      'sqlite_change_log.head_lag',\n      'Distance from the latest received commit to the SQLite change-log ' +\n        'head. The log commits before the forward, so this is zero except ' +\n        'within a transaction.',\n    ).addCallback(result => {\n      const lag = watermarkDistance(this.#receivedHead, this.#sqliteHead);\n      if (lag !== undefined) {\n        result.observe(lag);\n      }\n    });\n  }\n\n  /**\n   * Called as each message arrives, before it is written. For a sampled\n   * transaction, hashes the *received* stream and compares it with the hash the\n   * writer computes over what it *stored*.\n   */\n  messageReceived(\n    data: ChangeStreamData,\n    json: string,\n    validateHash = true,\n    storedChange = extractChangeSubstring(json, data[1].tag),\n  ): void {\n    const tag = data[1].tag;\n    if (data[0] === 'begin') {\n      if (this.#sourceHasher !== undefined) {\n        this.#invariantFailure(\n          'Received change-log begin while a source hash is already open',\n          {receivedWatermark: data[2].commitWatermark},\n        );\n      }\n      const watermark = data[2].commitWatermark;\n      this.#sourceHasher = validateHash\n        ? new ChangeLogTransactionHasher()\n        : undefined;\n      this.#sourcePos = 0;\n      this.#sourceCommitHash = undefined;\n      this.#sourceHasher?.add({\n        watermark,\n        pos: this.#sourcePos,\n        tag,\n        change: storedChange,\n        precommit: null,\n      });\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] === 'commit') {\n      this.#receivedHead = data[2].watermark;\n    }\n\n    if (!validateHash) {\n      return;\n    }\n\n    const watermark = this.#transactionWatermark;\n    if (this.#sourceHasher === undefined || watermark === undefined) {\n      this.#invariantFailure(\n        'Received change-log message without an observed transaction',\n        {tag},\n      );\n      return;\n    }\n\n    const commitWatermark =\n      data[0] === 'commit' ? data[2].watermark : watermark;\n    this.#sourceHasher.add({\n      watermark: commitWatermark,\n      pos: ++this.#sourcePos,\n      tag,\n      change: storedChange,\n      precommit: data[0] === 'commit' ? watermark : null,\n    });\n    if (data[0] === 'commit') {\n      this.#sourceCommitHash = this.#sourceHasher.digest();\n      this.#sourceHasher = undefined;\n    }\n  }\n\n  /** Called after the message has been written to the log. */\n  messageProcessed(\n    data: ChangeStreamData,\n    commit: ChangeLogCommit | null,\n    durationMs: number | undefined,\n  ): void {\n    const tag = data[1].tag;\n    if (durationMs !== undefined) {\n      this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'success'});\n    }\n\n    if (data[0] === 'begin') {\n      if (this.#transactionWatermark !== undefined) {\n        this.#invariantFailure('SQLite change-log transaction already open', {\n          openWatermark: this.#transactionWatermark,\n          receivedWatermark: data[2].commitWatermark,\n        });\n      }\n      this.#transactionWatermark = data[2].commitWatermark;\n      return;\n    }\n\n    if (tag === 'rollback') {\n      this.#recordRollback('upstream');\n      this.#transactionWatermark = undefined;\n      this.#resetSourceHash();\n      return;\n    }\n\n    if (data[0] !== 'commit') {\n      return;\n    }\n\n    const watermark = data[2].watermark;\n    if (commit !== null) {\n      this.#logCommitDuration.recordMs(commit.logCommitMs);\n    }\n    if (this.#transactionWatermark !== watermark) {\n      this.#invariantFailure(\n        'SQLite change-log commit does not match the observed begin',\n        {openWatermark: this.#transactionWatermark, commitWatermark: watermark},\n      );\n    }\n    if (commit === null) {\n      this.#invariantFailure(\n        'SQLite change-log commit result is missing writer statistics',\n        {commitWatermark: watermark},\n      );\n      this.#recordUnpaired(watermark, 'missing-sqlite-hash');\n    } else {\n      if (commit.watermark !== watermark) {\n        this.#invariantFailure(\n          'SQLite change-log commit result has an unexpected watermark',\n          {commitWatermark: watermark, resultWatermark: commit.watermark},\n        );\n      }\n      this.#rows += commit.stats.rows;\n      this.#estimatedBytes += commit.stats.estimatedBytes;\n      this.#transactionRows.record(commit.stats.rows);\n      if (\n        this.#sourceCommitHash !== undefined ||\n        commit.stats.hash !== undefined\n      ) {\n        this.#compareHashes(\n          watermark,\n          this.#sourceCommitHash,\n          commit.stats.hash,\n        );\n      }\n    }\n    if (watermark < this.#sqliteHead) {\n      this.#invariantFailure('SQLite change-log head regressed', {\n        previousHead: this.#sqliteHead,\n        commitWatermark: watermark,\n      });\n    }\n    this.#sqliteHead = watermark;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  messageFailed(\n    data: ChangeStreamData,\n    error: unknown,\n    durationMs: number | undefined,\n  ): void {\n    const tag = data[1].tag;\n    if (durationMs !== undefined) {\n      this.#messageProcessing.recordMs(durationMs, {tag, outcome: 'error'});\n    }\n    if (data[0] === 'commit' && this.#sourceCommitHash !== undefined) {\n      // No commit duration here: a failed commit produces no statistics, so\n      // there is nothing to attribute. The attempt is timed by\n      // `message_processing_duration{tag=\"commit\",outcome=\"error\"}` above.\n      this.#recordUnpaired(data[2].watermark, 'sqlite-commit-failed');\n    }\n    if (\n      error instanceof Error &&\n      error.name === 'ChangeLogStreamInvariantError'\n    ) {\n      this.#invariantFailure(error.message);\n    }\n    if (this.#transactionWatermark !== undefined || data[0] === 'begin') {\n      this.#recordRollback('processing-error');\n    }\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  /**\n   * A committed transaction folded schema changes into the cookie jar. Reported\n   * separately from {@link messageProcessed} because it costs a read of the\n   * cookie tables, which is only worth taking when they actually moved.\n   */\n  cookiesMutated(ops: readonly CookieOpTag[], rows: CookieRowCounts): void {\n    for (const op of ops) {\n      this.#cookieMutations++;\n      this.#cookieMutationCounter.add(1, {op});\n    }\n    this.#cookieRows = rows;\n  }\n\n  /**\n   * A missed truncate-above surfaces as a constraint violation on the writer's\n   * plain `INSERT`, which the fail-soft policy would otherwise absorb quietly.\n   * Classifying it as an invariant failure is what keeps a reconciliation bug\n   * from looking like a lost cache.\n   */\n  invariantFailure(message: string, details?: Record<string, unknown>): void {\n    this.#invariantFailure(message, details);\n  }\n\n  /** The source interrupted the stream mid-transaction. */\n  abort(): void {\n    if (this.#transactionWatermark !== undefined) {\n      this.#recordRollback('source-interruption');\n      this.#transactionWatermark = undefined;\n    }\n    this.#resetSourceHash();\n  }\n\n  /** Reconciliation moved the head, so the observer's totals are restated. */\n  reconciled(info: SQLiteChangeLogInfo): void {\n    this.#receivedHead = info.headWatermark;\n    this.#sqliteHead = info.headWatermark;\n    this.#rows = info.rows;\n    this.#estimatedBytes = info.estimatedBytes;\n    this.#cookieRows = info.cookieRows;\n    this.#transactionWatermark = undefined;\n    this.#resetSourceHash();\n  }\n\n  state(): SQLiteChangeLogObservabilityState {\n    return {\n      receivedHead: this.#receivedHead,\n      sqliteHead: this.#sqliteHead,\n      headLag: watermarkDistance(this.#receivedHead, this.#sqliteHead),\n      rows: this.#rows,\n      estimatedBytes: this.#estimatedBytes,\n      cookieRows: this.#cookieRows,\n      cookieMutations: this.#cookieMutations,\n      rollbacks: this.#rollbacks,\n      invariantFailures: this.#invariantFailures,\n      hashMatches: this.#hashMatches,\n      hashMismatches: this.#hashMismatches,\n      hashUnpaired: this.#hashUnpaired,\n    };\n  }\n\n  #recordRollback(\n    reason: 'upstream' | 'processing-error' | 'source-interruption',\n  ) {\n    this.#rollbacks++;\n    this.#rollbackCounter.add(1, {reason});\n  }\n\n  #invariantFailure(message: string, details?: Record<string, unknown>) {\n    this.#invariantFailures++;\n    this.#invariantFailureCounter.add(1);\n    this.#lc.error?.(message, details);\n  }\n\n  #compareHashes(\n    watermark: string,\n    sourceHash: string | undefined,\n    sqliteHash: string | undefined,\n  ): void {\n    if (sourceHash === undefined) {\n      this.#recordUnpaired(watermark, 'missing-source-hash');\n      return;\n    }\n    if (sqliteHash === undefined) {\n      this.#recordUnpaired(watermark, 'missing-sqlite-hash');\n      return;\n    }\n    if (sourceHash === sqliteHash) {\n      this.#hashMatches++;\n      this.#hashComparisonCounter.add(1, {outcome: 'match'});\n      return;\n    }\n    this.#hashMismatches++;\n    this.#hashComparisonCounter.add(1, {outcome: 'mismatch'});\n    this.#lc.error?.('SQLite change-log transaction hash mismatch', {\n      sqliteChangeLogHashComparison: {\n        watermark,\n        sourceHash: sourceHash.slice(0, 16),\n        sqliteHash: sqliteHash.slice(0, 16),\n      },\n    });\n  }\n\n  #recordUnpaired(\n    watermark: string,\n    reason:\n      | 'missing-source-hash'\n      | 'missing-sqlite-hash'\n      | 'sqlite-commit-failed',\n  ): void {\n    this.#hashUnpaired++;\n    this.#hashUnpairedCounter.add(1, {reason});\n    this.#lc.warn?.('SQLite change-log transaction hash was not compared', {\n      sqliteChangeLogHashComparison: {watermark, reason},\n    });\n  }\n\n  #resetSourceHash(): void {\n    this.#sourceHasher = undefined;\n    this.#sourcePos = 0;\n    this.#sourceCommitHash = undefined;\n  }\n}\n\nfunction watermarkValue(watermark: string): number | undefined {\n  try {\n    const value = Number(versionFromLexi(watermark));\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n\nfunction watermarkDistance(\n  receivedHead: string,\n  sqliteHead: string,\n): number | undefined {\n  if (receivedHead <= sqliteHead) {\n    return 0;\n  }\n  try {\n    const distance =\n      versionFromLexi(receivedHead) - versionFromLexi(sqliteHead);\n    const value = Number(distance);\n    return Number.isFinite(value) ? value : undefined;\n  } catch {\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA,SAAgB,+BACd,IACA,QACM;CACN,QAAQ,OAAO,QAAf;EACE,KAAK;GACH,mBACE,WACA,8CACA,kHAEF,EAAE,IAAI,OAAO,IAAI;GACjB;EACF,KAAK;GACH,mBACE,WACA,qCACA,mIAEF,EAAE,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;GAChC;EACF,KAAK,QACH;EACF,SACE,YAAY,MAAM;CACtB;CACA,IAAI,OAAO,cAKT,mBACE,WACA,mCACA,sGAEF,EAAE,IAAI,GAAG,EAAC,QAAQ,OAAO,OAAM,CAAC;CAElC,GAAG,QAAQ,oCAAoC,EAC7C,0BAA0B,OAC5B,CAAC;AACH;;;;;;;;AAyBA,SAAgB,8BACd,WAC4B;CAC5B,MAAM,OAAO,kBAAkB,SAAS;CACxC,MAAM,EAAC,cAAc,kBAAiB,oBAAoB,SAAS;CACnE,IAAI,iBAAiB,QAAQ,kBAAkB,MAC7C,MAAM,IAAI,MAAM,yDAAyD;CAE3E,OAAO;EAAC,GAAG;EAAM;EAAc;CAAa;AAC9C;;;;;;;AAQA,SAAgB,uBACd,WACqB;CACrB,MAAM,UAAU,8BAA8B,SAAS;CACvD,MAAM,YAAY,UACf,QAAgB;;;gBAGL,wBAAwB;KACnC,EACA,IAA4C;CAE/C,OAAO;EACL,GAAG;EACH,MAAM,UAAU;EAChB,gBAAgB,UAAU;EAC1B,YAAY,oBAAoB,SAAS;CAC3C;AACF;AAEA,SAAgB,0BACd,IACA,MACA,MACM;CACN,GAAG,OAAO,6BAA6B,EACrC,iBAAiB;EACf;EACA,OAAO,KAAK;EACZ,YAAY,KAAK;EACjB,WAAW,KAAK;EAChB,eAAe,KAAK;EACpB,eAAe,KAAK;EACpB,YAAY,KAAK;EACjB,cAAc,KAAK;EACnB,eAAe,KAAK;CACtB,EACF,CAAC;AACH;;;;;;;;;;AAWA,eAAsB,gBACpB,IACA,MACiB;CAajB,QAAO,MAZa,QAAQ,IAC1B;EAAC;EAAM,GAAG,KAAK;EAAO,GAAG,KAAK;CAAM,EAAE,IAAI,OAAM,SAAQ;EACtD,IAAI;GACF,QAAQ,MAAM,KAAK,IAAI,GAAG;EAC5B,SAAS,GAAG;GACV,IAAK,EAA4B,SAAS,UACxC,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;GAExD,OAAO;EACT;CACF,CAAC,CACH,GACa,QAAQ,OAAO,SAAS,QAAQ,MAAM,CAAC;AACtD;AA4BA,IAAM,0BAA0B;CAAC;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAM;AAAI;;;;;;;AAQ1E,IAAa,0BAAb,MAAqC;CACnC;CACA,qBAA8B,4BAC5B,WACA,iDACA,uEACF;CAMA,qBAA8B,4BAC5B,WACA,yCACA,+KAGF;CACA,mBAA4B,0BAC1B,WACA,sCACA;EACE,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CACF;CACA,mBAA4B,mBAC1B,WACA,+BACA,2DACF;CACA,2BAAoC,mBAClC,WACA,wCACA,uDACF;CACA,yBAAkC,mBAChC,WACA,sCACA,iHACF;CACA,uBAAgC,mBAC9B,WACA,mCACA,2FACF;CAEA,yBAAkC,mBAChC,WACA,sCACA,uEACF;CAEA;CACA;CACA;CACA;CACA;CACA,mBAAmB;CACnB;CACA;CACA,aAAa;CACb;CACA,aAAa;CACb,qBAAqB;CACrB,eAAe;CACf,kBAAkB;CAClB,gBAAgB;CAEhB,YAAY,IAAgB,MAA2B;EACrD,KAAKA,MAAM,GAAG,YAAY,aAAa,4BAA4B;EAGnE,KAAKS,gBAAgB,KAAK;EAC1B,KAAKC,cAAc,KAAK;EACxB,KAAKC,QAAQ,KAAK;EAClB,KAAKC,kBAAkB,KAAK;EAC5B,KAAKC,cAAc,KAAK;EAExB,iBACE,WACA,0BACA,yCACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKF,KAAK,CAAC;EAGlD,iBACE,WACA,iCACA,iEACF,EAAE,aAAY,WAAU;GACtB,OAAO,QAAQ,KAAKE,YAAY,eAAe,EAAC,OAAO,gBAAe,CAAC;GACvE,OAAO,QAAQ,KAAKA,YAAY,aAAa,EAAC,OAAO,cAAa,CAAC;EACrE,CAAC;EACD,iBAAiB,WAAW,oCAAoC;GAC9D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKD,eAAe,CAAC;EAC7D,iBACE,WACA,0BACA,oEACF,EAAE,aAAY,WAAU;GACtB,MAAM,OAAO,eAAe,KAAKF,WAAW;GAC5C,IAAI,SAAS,KAAA,GACX,OAAO,QAAQ,IAAI;EAEvB,CAAC;EACD,iBACE,WACA,8BACA,0JAGF,EAAE,aAAY,WAAU;GACtB,MAAM,MAAM,kBAAkB,KAAKD,eAAe,KAAKC,WAAW;GAClE,IAAI,QAAQ,KAAA,GACV,OAAO,QAAQ,GAAG;EAEtB,CAAC;CACH;;;;;;CAOA,gBACE,MACA,MACA,eAAe,MACf,eAAe,uBAAuB,MAAM,KAAK,GAAG,GAAG,GACjD;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAKI,kBAAkB,KAAA,GACzB,KAAKC,kBACH,iEACA,EAAC,mBAAmB,KAAK,GAAG,gBAAe,CAC7C;GAEF,MAAM,YAAY,KAAK,GAAG;GAC1B,KAAKD,gBAAgB,eACjB,IAAI,2BAA2B,IAC/B,KAAA;GACJ,KAAKE,aAAa;GAClB,KAAKC,oBAAoB,KAAA;GACzB,KAAKH,eAAe,IAAI;IACtB;IACA,KAAK,KAAKE;IACV;IACA,QAAQ;IACR,WAAW;GACb,CAAC;GACD;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAKE,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd,KAAKT,gBAAgB,KAAK,GAAG;EAG/B,IAAI,CAAC,cACH;EAGF,MAAM,YAAY,KAAKU;EACvB,IAAI,KAAKL,kBAAkB,KAAA,KAAa,cAAc,KAAA,GAAW;GAC/D,KAAKC,kBACH,+DACA,EAAC,IAAG,CACN;GACA;EACF;EAEA,MAAM,kBACJ,KAAK,OAAO,WAAW,KAAK,GAAG,YAAY;EAC7C,KAAKD,cAAc,IAAI;GACrB,WAAW;GACX,KAAK,EAAE,KAAKE;GACZ;GACA,QAAQ;GACR,WAAW,KAAK,OAAO,WAAW,YAAY;EAChD,CAAC;EACD,IAAI,KAAK,OAAO,UAAU;GACxB,KAAKC,oBAAoB,KAAKH,cAAc,OAAO;GACnD,KAAKA,gBAAgB,KAAA;EACvB;CACF;;CAGA,iBACE,MACA,QACA,YACM;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,IAAI,eAAe,KAAA,GACjB,KAAKb,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAS,CAAC;EAGxE,IAAI,KAAK,OAAO,SAAS;GACvB,IAAI,KAAKkB,0BAA0B,KAAA,GACjC,KAAKJ,kBAAkB,8CAA8C;IACnE,eAAe,KAAKI;IACpB,mBAAmB,KAAK,GAAG;GAC7B,CAAC;GAEH,KAAKA,wBAAwB,KAAK,GAAG;GACrC;EACF;EAEA,IAAI,QAAQ,YAAY;GACtB,KAAKC,gBAAgB,UAAU;GAC/B,KAAKD,wBAAwB,KAAA;GAC7B,KAAKD,iBAAiB;GACtB;EACF;EAEA,IAAI,KAAK,OAAO,UACd;EAGF,MAAM,YAAY,KAAK,GAAG;EAC1B,IAAI,WAAW,MACb,KAAKhB,mBAAmB,SAAS,OAAO,WAAW;EAErD,IAAI,KAAKiB,0BAA0B,WACjC,KAAKJ,kBACH,8DACA;GAAC,eAAe,KAAKI;GAAuB,iBAAiB;EAAS,CACxE;EAEF,IAAI,WAAW,MAAM;GACnB,KAAKJ,kBACH,gEACA,EAAC,iBAAiB,UAAS,CAC7B;GACA,KAAKM,gBAAgB,WAAW,qBAAqB;EACvD,OAAO;GACL,IAAI,OAAO,cAAc,WACvB,KAAKN,kBACH,+DACA;IAAC,iBAAiB;IAAW,iBAAiB,OAAO;GAAS,CAChE;GAEF,KAAKJ,SAAS,OAAO,MAAM;GAC3B,KAAKC,mBAAmB,OAAO,MAAM;GACrC,KAAKT,iBAAiB,OAAO,OAAO,MAAM,IAAI;GAC9C,IACE,KAAKc,sBAAsB,KAAA,KAC3B,OAAO,MAAM,SAAS,KAAA,GAEtB,KAAKK,eACH,WACA,KAAKL,mBACL,OAAO,MAAM,IACf;EAEJ;EACA,IAAI,YAAY,KAAKP,aACnB,KAAKK,kBAAkB,oCAAoC;GACzD,cAAc,KAAKL;GACnB,iBAAiB;EACnB,CAAC;EAEH,KAAKA,cAAc;EACnB,KAAKS,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;CAEA,cACE,MACA,OACA,YACM;EACN,MAAM,MAAM,KAAK,GAAG;EACpB,IAAI,eAAe,KAAA,GACjB,KAAKjB,mBAAmB,SAAS,YAAY;GAAC;GAAK,SAAS;EAAO,CAAC;EAEtE,IAAI,KAAK,OAAO,YAAY,KAAKgB,sBAAsB,KAAA,GAIrD,KAAKI,gBAAgB,KAAK,GAAG,WAAW,sBAAsB;EAEhE,IACE,iBAAiB,SACjB,MAAM,SAAS,iCAEf,KAAKN,kBAAkB,MAAM,OAAO;EAEtC,IAAI,KAAKI,0BAA0B,KAAA,KAAa,KAAK,OAAO,SAC1D,KAAKC,gBAAgB,kBAAkB;EAEzC,KAAKD,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;;;;;;CAOA,eAAe,KAA6B,MAA6B;EACvE,KAAK,MAAM,MAAM,KAAK;GACpB,KAAKK;GACL,KAAKf,uBAAuB,IAAI,GAAG,EAAC,GAAE,CAAC;EACzC;EACA,KAAKK,cAAc;CACrB;;;;;;;CAQA,iBAAiB,SAAiB,SAAyC;EACzE,KAAKE,kBAAkB,SAAS,OAAO;CACzC;;CAGA,QAAc;EACZ,IAAI,KAAKI,0BAA0B,KAAA,GAAW;GAC5C,KAAKC,gBAAgB,qBAAqB;GAC1C,KAAKD,wBAAwB,KAAA;EAC/B;EACA,KAAKD,iBAAiB;CACxB;;CAGA,WAAW,MAAiC;EAC1C,KAAKT,gBAAgB,KAAK;EAC1B,KAAKC,cAAc,KAAK;EACxB,KAAKC,QAAQ,KAAK;EAClB,KAAKC,kBAAkB,KAAK;EAC5B,KAAKC,cAAc,KAAK;EACxB,KAAKM,wBAAwB,KAAA;EAC7B,KAAKD,iBAAiB;CACxB;CAEA,QAA2C;EACzC,OAAO;GACL,cAAc,KAAKT;GACnB,YAAY,KAAKC;GACjB,SAAS,kBAAkB,KAAKD,eAAe,KAAKC,WAAW;GAC/D,MAAM,KAAKC;GACX,gBAAgB,KAAKC;GACrB,YAAY,KAAKC;GACjB,iBAAiB,KAAKU;GACtB,WAAW,KAAKC;GAChB,mBAAmB,KAAKC;GACxB,aAAa,KAAKC;GAClB,gBAAgB,KAAKC;GACrB,cAAc,KAAKC;EACrB;CACF;CAEA,gBACE,QACA;EACA,KAAKJ;EACL,KAAKpB,iBAAiB,IAAI,GAAG,EAAC,OAAM,CAAC;CACvC;CAEA,kBAAkB,SAAiB,SAAmC;EACpE,KAAKqB;EACL,KAAKpB,yBAAyB,IAAI,CAAC;EACnC,KAAKL,IAAI,QAAQ,SAAS,OAAO;CACnC;CAEA,eACE,WACA,YACA,YACM;EACN,IAAI,eAAe,KAAA,GAAW;GAC5B,KAAKqB,gBAAgB,WAAW,qBAAqB;GACrD;EACF;EACA,IAAI,eAAe,KAAA,GAAW;GAC5B,KAAKA,gBAAgB,WAAW,qBAAqB;GACrD;EACF;EACA,IAAI,eAAe,YAAY;GAC7B,KAAKK;GACL,KAAKpB,uBAAuB,IAAI,GAAG,EAAC,SAAS,QAAO,CAAC;GACrD;EACF;EACA,KAAKqB;EACL,KAAKrB,uBAAuB,IAAI,GAAG,EAAC,SAAS,WAAU,CAAC;EACxD,KAAKN,IAAI,QAAQ,+CAA+C,EAC9D,+BAA+B;GAC7B;GACA,YAAY,WAAW,MAAM,GAAG,EAAE;GAClC,YAAY,WAAW,MAAM,GAAG,EAAE;EACpC,EACF,CAAC;CACH;CAEA,gBACE,WACA,QAIM;EACN,KAAK4B;EACL,KAAKrB,qBAAqB,IAAI,GAAG,EAAC,OAAM,CAAC;EACzC,KAAKP,IAAI,OAAO,uDAAuD,EACrE,+BAA+B;GAAC;GAAW;EAAM,EACnD,CAAC;CACH;CAEA,mBAAyB;EACvB,KAAKc,gBAAgB,KAAA;EACrB,KAAKE,aAAa;EAClB,KAAKC,oBAAoB,KAAA;CAC3B;AACF;AAEA,SAAS,eAAe,WAAuC;CAC7D,IAAI;EACF,MAAM,QAAQ,OAAO,gBAAgB,SAAS,CAAC;EAC/C,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF;AAEA,SAAS,kBACP,cACA,YACoB;CACpB,IAAI,gBAAgB,YAClB,OAAO;CAET,IAAI;EACF,MAAM,WACJ,gBAAgB,YAAY,IAAI,gBAAgB,UAAU;EAC5D,MAAM,QAAQ,OAAO,QAAQ;EAC7B,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;CAC1C,QAAQ;EACN;CACF;AACF"}