@rocicorp/zero 1.10.0-canary.1 → 1.10.0-canary.3

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-reservations.js","names":["#lc","#backupConfig","#onClose","#reservations","#close","#downstream","#watermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {BackupConfig} from './change-streamer-service.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport class SnapshotReservations {\n  readonly #lc: LogContext;\n  readonly #backupConfig: BackupConfig;\n  readonly #onClose: ((taskID: string) => void) | undefined;\n  readonly #reservations = new Map<string, Reservation>();\n\n  constructor(\n    lc: LogContext,\n    backupConfig: BackupConfig,\n    onClose?: ((taskID: string) => void) | undefined,\n  ) {\n    this.#lc = lc.withContext('component', 'snapshot-reserver');\n    this.#backupConfig = backupConfig;\n    this.#onClose = onClose;\n  }\n\n  open(taskID: string): Source<SnapshotMessage> {\n    this.close(taskID);\n\n    const instanceID = {};\n    const downstream = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.#close(taskID, instanceID),\n    });\n    this.#reservations.set(taskID, new Reservation(instanceID, downstream));\n    this.#lc.info?.(`created snasphot reservation for ${taskID}`);\n    return downstream;\n  }\n\n  close(taskID: string) {\n    this.#close(taskID, undefined);\n  }\n\n  #close(taskID: string, cancelledInstanceID: InstanceID | undefined) {\n    const res = this.#reservations.get(taskID);\n    if (\n      res &&\n      (!cancelledInstanceID || res.instanceID === cancelledInstanceID)\n    ) {\n      // Note: delete first, so that the reservation is gone when close() is called.\n      this.#reservations.delete(taskID);\n      this.#onClose?.(taskID);\n      res.close();\n\n      const duration = Date.now() - res.startTime.getTime();\n      this.#lc.info?.(\n        `ended snapshot reservation for ${taskID} (${duration} ms)`,\n      );\n      litestreamSnapshotReservationDuration().recordMs(duration, {\n        ...litestreamMonitorMetricAttrs(\n          this.#backupConfig.backupURL,\n          this.#backupConfig.litestreamVersion,\n          'view_syncer',\n        ),\n        result:\n          cancelledInstanceID || !res.confirmed() ? 'cancelled' : 'closed',\n      });\n    }\n  }\n\n  confirmationsRequired() {\n    for (const res of this.#reservations.values()) {\n      if (!res.confirmed()) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  confirm(replicaVersion: string, minWatermark: string) {\n    for (const [taskID, res] of this.#reservations.entries()) {\n      if (!res.confirmed()) {\n        this.#lc.info?.(\n          `reserving change-log entries since ${minWatermark} for ${taskID}`,\n        );\n        res.confirm(this.#backupConfig.backupURL, replicaVersion, minWatermark);\n      }\n    }\n  }\n\n  getReservedWatermarks() {\n    return Array.from(\n      this.#reservations.values(),\n      ({reservedWatermark}) => reservedWatermark,\n    ).filter(watermark => watermark !== null);\n  }\n}\n\ntype InstanceID = {};\n\nclass Reservation {\n  readonly instanceID: InstanceID;\n  readonly startTime: Date = new Date();\n  readonly #downstream: Subscription<SnapshotMessage>;\n  #watermark: string | null = null;\n\n  constructor(\n    instanceID: InstanceID,\n    downstream: Subscription<SnapshotMessage>,\n  ) {\n    this.instanceID = instanceID;\n    this.#downstream = downstream;\n  }\n\n  get reservedWatermark() {\n    return this.#watermark;\n  }\n\n  confirmed() {\n    return this.#watermark !== null;\n  }\n\n  confirm(backupURL: string, replicaVersion: string, minWatermark: string) {\n    if (this.#watermark === null) {\n      if (this.#downstream.active) {\n        this.#downstream.push([\n          'status',\n          {tag: 'status', backupURL, replicaVersion, minWatermark},\n        ]);\n      }\n      this.#watermark = minWatermark;\n    }\n  }\n\n  close() {\n    this.#downstream.cancel();\n  }\n}\n"],"mappings":";;;AAUA,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA,gCAAyB,IAAI,IAAyB;CAEtD,YACE,IACA,cACA,SACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKC,gBAAgB;EACrB,KAAKC,WAAW;CAClB;CAEA,KAAK,QAAyC;EAC5C,KAAK,MAAM,MAAM;EAEjB,MAAM,aAAa,CAAC;EACpB,MAAM,aAAa,aAAa,OAAwB,EACtD,eAAe,KAAKE,OAAO,QAAQ,UAAU,EAC/C,CAAC;EACD,KAAKD,cAAc,IAAI,QAAQ,IAAI,YAAY,YAAY,UAAU,CAAC;EACtE,KAAKH,IAAI,OAAO,oCAAoC,QAAQ;EAC5D,OAAO;CACT;CAEA,MAAM,QAAgB;EACpB,KAAKI,OAAO,QAAQ,KAAA,CAAS;CAC/B;CAEA,OAAO,QAAgB,qBAA6C;EAClE,MAAM,MAAM,KAAKD,cAAc,IAAI,MAAM;EACzC,IACE,QACC,CAAC,uBAAuB,IAAI,eAAe,sBAC5C;GAEA,KAAKA,cAAc,OAAO,MAAM;GAChC,KAAKD,WAAW,MAAM;GACtB,IAAI,MAAM;GAEV,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ;GACpD,KAAKF,IAAI,OACP,kCAAkC,OAAO,IAAI,SAAS,KACxD;GACA,sCAAsC,EAAE,SAAS,UAAU;IACzD,GAAG,6BACD,KAAKC,cAAc,WACnB,KAAKA,cAAc,mBACnB,aACF;IACA,QACE,uBAAuB,CAAC,IAAI,UAAU,IAAI,cAAc;GAC5D,CAAC;EACH;CACF;CAEA,wBAAwB;EACtB,KAAK,MAAM,OAAO,KAAKE,cAAc,OAAO,GAC1C,IAAI,CAAC,IAAI,UAAU,GACjB,OAAO;EAGX,OAAO;CACT;CAEA,QAAQ,gBAAwB,cAAsB;EACpD,KAAK,MAAM,CAAC,QAAQ,QAAQ,KAAKA,cAAc,QAAQ,GACrD,IAAI,CAAC,IAAI,UAAU,GAAG;GACpB,KAAKH,IAAI,OACP,sCAAsC,aAAa,OAAO,QAC5D;GACA,IAAI,QAAQ,KAAKC,cAAc,WAAW,gBAAgB,YAAY;EACxE;CAEJ;CAEA,wBAAwB;EACtB,OAAO,MAAM,KACX,KAAKE,cAAc,OAAO,IACzB,EAAC,wBAAuB,iBAC3B,EAAE,QAAO,cAAa,cAAc,IAAI;CAC1C;AACF;AAIA,IAAM,cAAN,MAAkB;CAChB;CACA,4BAA2B,IAAI,KAAK;CACpC;CACA,aAA4B;CAE5B,YACE,YACA,YACA;EACA,KAAK,aAAa;EAClB,KAAKE,cAAc;CACrB;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAKC;CACd;CAEA,YAAY;EACV,OAAO,KAAKA,eAAe;CAC7B;CAEA,QAAQ,WAAmB,gBAAwB,cAAsB;EACvE,IAAI,KAAKA,eAAe,MAAM;GAC5B,IAAI,KAAKD,YAAY,QACnB,KAAKA,YAAY,KAAK,CACpB,UACA;IAAC,KAAK;IAAU;IAAW;IAAgB;GAAY,CACzD,CAAC;GAEH,KAAKC,aAAa;EACpB;CACF;CAEA,QAAQ;EACN,KAAKD,YAAY,OAAO;CAC1B;AACF"}
+{"version":3,"file":"snapshot-reservations.js","names":["#lc","#backupConfig","#onClose","#reservations","#close","#metricAttrs","#downstream","#watermark","#delayNoted"],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationConfirmDuration,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {BackupConfig} from './change-streamer-service.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\nimport type {ChangeLogReadSource} from './sqlite-change-log-read-router.ts';\n\nexport class SnapshotReservations {\n  readonly #lc: LogContext;\n  readonly #backupConfig: BackupConfig;\n  readonly #onClose: ((taskID: string) => void) | undefined;\n  readonly #reservations = new Map<string, Reservation>();\n\n  constructor(\n    lc: LogContext,\n    backupConfig: BackupConfig,\n    onClose?: ((taskID: string) => void) | undefined,\n  ) {\n    this.#lc = lc.withContext('component', 'snapshot-reserver');\n    this.#backupConfig = backupConfig;\n    this.#onClose = onClose;\n  }\n\n  open(taskID: string): Source<SnapshotMessage> {\n    this.close(taskID);\n\n    const instanceID = {};\n    const downstream = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.#close(taskID, instanceID),\n    });\n    this.#reservations.set(taskID, new Reservation(instanceID, downstream));\n    this.#lc.info?.(`created snasphot reservation for ${taskID}`);\n    return downstream;\n  }\n\n  close(taskID: string) {\n    this.#close(taskID, undefined);\n  }\n\n  isCurrent(taskID: string, source: Source<SnapshotMessage>): boolean {\n    return this.#reservations.get(taskID)?.owns(source) ?? false;\n  }\n\n  #metricAttrs() {\n    return litestreamMonitorMetricAttrs(\n      this.#backupConfig.backupURL,\n      this.#backupConfig.litestreamVersion,\n      'view_syncer',\n    );\n  }\n\n  #close(taskID: string, cancelledInstanceID: InstanceID | undefined) {\n    const res = this.#reservations.get(taskID);\n    if (\n      res &&\n      (!cancelledInstanceID || res.instanceID === cancelledInstanceID)\n    ) {\n      // Note: delete first, so that the reservation is gone when close() is called.\n      this.#reservations.delete(taskID);\n      this.#onClose?.(taskID);\n      res.close();\n\n      const duration = Date.now() - res.startTime.getTime();\n      this.#lc.info?.(\n        `ended snapshot reservation for ${taskID} (${duration} ms)`,\n      );\n      // `result` reports how the reservation ended and `confirmed` whether it\n      // ever received its bounds. They are separate dimensions: a follower\n      // that gives up while waiting for a confirmation is\n      // `result=cancelled, confirmed=false`, which a single collapsed\n      // attribute cannot distinguish from a client that simply went away.\n      litestreamSnapshotReservationDuration().recordMs(duration, {\n        ...this.#metricAttrs(),\n        result: cancelledInstanceID ? 'cancelled' : 'closed',\n        confirmed: res.confirmed(),\n      });\n    }\n  }\n\n  confirmationsRequired() {\n    for (const res of this.#reservations.values()) {\n      if (!res.confirmed()) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  unconfirmedTaskIDs(): string[] {\n    return [...this.#reservations.entries()]\n      .filter(([, reservation]) => !reservation.confirmed())\n      .map(([taskID]) => taskID);\n  }\n\n  /** Confirms one reservation with the bounds of its pinned read source. */\n  confirmFor(\n    taskID: string,\n    replicaVersion: string,\n    minWatermark: string,\n    source: ChangeLogReadSource,\n  ): void {\n    const res = this.#reservations.get(taskID);\n    if (res && !res.confirmed()) {\n      this.#lc.info?.(\n        `reserving change-log entries since ${minWatermark} for ${taskID}`,\n      );\n      res.confirm(this.#backupConfig.backupURL, replicaVersion, minWatermark);\n      // Measured from `open()`, not from the first confirmation attempt: what\n      // matters is how long the follower waited before it could restore.\n      litestreamSnapshotReservationConfirmDuration().recordMs(\n        Date.now() - res.startTime.getTime(),\n        {...this.#metricAttrs(), source},\n      );\n    }\n  }\n\n  /**\n   * Notes that a confirmation was deferred, returning true only the first time\n   * for this reservation. Confirmation is retried on every backup, so an\n   * undeduplicated count would measure backups rather than delayed followers.\n   */\n  noteConfirmationDelayed(taskID: string): boolean {\n    return this.#reservations.get(taskID)?.noteDelayed() ?? false;\n  }\n\n  getReservedWatermarks() {\n    return Array.from(\n      this.#reservations.values(),\n      ({reservedWatermark}) => reservedWatermark,\n    ).filter(watermark => watermark !== null);\n  }\n}\n\ntype InstanceID = {};\n\nclass Reservation {\n  readonly instanceID: InstanceID;\n  readonly startTime: Date = new Date();\n  readonly #downstream: Subscription<SnapshotMessage>;\n  #watermark: string | null = null;\n  #delayNoted = false;\n\n  constructor(\n    instanceID: InstanceID,\n    downstream: Subscription<SnapshotMessage>,\n  ) {\n    this.instanceID = instanceID;\n    this.#downstream = downstream;\n  }\n\n  get reservedWatermark() {\n    return this.#watermark;\n  }\n\n  confirmed() {\n    return this.#watermark !== null;\n  }\n\n  owns(source: Source<SnapshotMessage>): boolean {\n    return this.#downstream === source;\n  }\n\n  noteDelayed(): boolean {\n    if (this.#delayNoted) {\n      return false;\n    }\n    this.#delayNoted = true;\n    return true;\n  }\n\n  confirm(backupURL: string, replicaVersion: string, minWatermark: string) {\n    if (this.#watermark === null) {\n      if (this.#downstream.active) {\n        this.#downstream.push([\n          'status',\n          {tag: 'status', backupURL, replicaVersion, minWatermark},\n        ]);\n      }\n      this.#watermark = minWatermark;\n    }\n  }\n\n  close() {\n    this.#downstream.cancel();\n  }\n}\n"],"mappings":";;;AAYA,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA,gCAAyB,IAAI,IAAyB;CAEtD,YACE,IACA,cACA,SACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKC,gBAAgB;EACrB,KAAKC,WAAW;CAClB;CAEA,KAAK,QAAyC;EAC5C,KAAK,MAAM,MAAM;EAEjB,MAAM,aAAa,CAAC;EACpB,MAAM,aAAa,aAAa,OAAwB,EACtD,eAAe,KAAKE,OAAO,QAAQ,UAAU,EAC/C,CAAC;EACD,KAAKD,cAAc,IAAI,QAAQ,IAAI,YAAY,YAAY,UAAU,CAAC;EACtE,KAAKH,IAAI,OAAO,oCAAoC,QAAQ;EAC5D,OAAO;CACT;CAEA,MAAM,QAAgB;EACpB,KAAKI,OAAO,QAAQ,KAAA,CAAS;CAC/B;CAEA,UAAU,QAAgB,QAA0C;EAClE,OAAO,KAAKD,cAAc,IAAI,MAAM,GAAG,KAAK,MAAM,KAAK;CACzD;CAEA,eAAe;EACb,OAAO,6BACL,KAAKF,cAAc,WACnB,KAAKA,cAAc,mBACnB,aACF;CACF;CAEA,OAAO,QAAgB,qBAA6C;EAClE,MAAM,MAAM,KAAKE,cAAc,IAAI,MAAM;EACzC,IACE,QACC,CAAC,uBAAuB,IAAI,eAAe,sBAC5C;GAEA,KAAKA,cAAc,OAAO,MAAM;GAChC,KAAKD,WAAW,MAAM;GACtB,IAAI,MAAM;GAEV,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ;GACpD,KAAKF,IAAI,OACP,kCAAkC,OAAO,IAAI,SAAS,KACxD;GAMA,sCAAsC,EAAE,SAAS,UAAU;IACzD,GAAG,KAAKK,aAAa;IACrB,QAAQ,sBAAsB,cAAc;IAC5C,WAAW,IAAI,UAAU;GAC3B,CAAC;EACH;CACF;CAEA,wBAAwB;EACtB,KAAK,MAAM,OAAO,KAAKF,cAAc,OAAO,GAC1C,IAAI,CAAC,IAAI,UAAU,GACjB,OAAO;EAGX,OAAO;CACT;CAEA,qBAA+B;EAC7B,OAAO,CAAC,GAAG,KAAKA,cAAc,QAAQ,CAAC,EACpC,QAAQ,GAAG,iBAAiB,CAAC,YAAY,UAAU,CAAC,EACpD,KAAK,CAAC,YAAY,MAAM;CAC7B;;CAGA,WACE,QACA,gBACA,cACA,QACM;EACN,MAAM,MAAM,KAAKA,cAAc,IAAI,MAAM;EACzC,IAAI,OAAO,CAAC,IAAI,UAAU,GAAG;GAC3B,KAAKH,IAAI,OACP,sCAAsC,aAAa,OAAO,QAC5D;GACA,IAAI,QAAQ,KAAKC,cAAc,WAAW,gBAAgB,YAAY;GAGtE,6CAA6C,EAAE,SAC7C,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ,GACnC;IAAC,GAAG,KAAKI,aAAa;IAAG;GAAM,CACjC;EACF;CACF;;;;;;CAOA,wBAAwB,QAAyB;EAC/C,OAAO,KAAKF,cAAc,IAAI,MAAM,GAAG,YAAY,KAAK;CAC1D;CAEA,wBAAwB;EACtB,OAAO,MAAM,KACX,KAAKA,cAAc,OAAO,IACzB,EAAC,wBAAuB,iBAC3B,EAAE,QAAO,cAAa,cAAc,IAAI;CAC1C;AACF;AAIA,IAAM,cAAN,MAAkB;CAChB;CACA,4BAA2B,IAAI,KAAK;CACpC;CACA,aAA4B;CAC5B,cAAc;CAEd,YACE,YACA,YACA;EACA,KAAK,aAAa;EAClB,KAAKG,cAAc;CACrB;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAKC;CACd;CAEA,YAAY;EACV,OAAO,KAAKA,eAAe;CAC7B;CAEA,KAAK,QAA0C;EAC7C,OAAO,KAAKD,gBAAgB;CAC9B;CAEA,cAAuB;EACrB,IAAI,KAAKE,aACP,OAAO;EAET,KAAKA,cAAc;EACnB,OAAO;CACT;CAEA,QAAQ,WAAmB,gBAAwB,cAAsB;EACvE,IAAI,KAAKD,eAAe,MAAM;GAC5B,IAAI,KAAKD,YAAY,QACnB,KAAKA,YAAY,KAAK,CACpB,UACA;IAAC,KAAK;IAAU;IAAW;IAAgB;GAAY,CACzD,CAAC;GAEH,KAAKC,aAAa;EACpB;CACF;CAEA,QAAQ;EACN,KAAKD,YAAY,OAAO;CAC1B;AACF"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts

@@ -24,6 +24,30 @@ export type SQLiteChangeLogCatchupOptions = {
     cleanupGuard?: SQLiteChangeLogCleanupGuard | undefined;
     sleep?: typeof sleep | undefined;
     now?: (() => number) | undefined;
+    /** Opens slice 11's process-local breaker after registration has committed. */
+    onFailure?: ((failure: SQLiteChangeLogCatchupFailure) => void) | undefined;
+};
+export type SQLiteChangeLogCatchupRequest = {
+    /**
+     * Whether the routing warm-up gate classified the log as warm *for this
+     * subscriber*. The coordinator is opened once and reused, so this cannot be
+     * a property of it: `coldReadPercent` serves some subscribers from a log
+     * that is still inside its warm-up window, and the catchup metrics have to
+     * separate those from steady-state serving.
+     */
+    readonly logWarm?: boolean | undefined;
+};
+export type SQLiteChangeLogCatchupFailure = 'barrier-timeout' | 'reader-error';
+export type SQLiteChangeLogCatchupRegistration = {
+    readonly kind: 'registered';
+} | {
+    readonly kind: 'uncovered';
+    readonly minWatermark: string;
+} | {
+    readonly kind: 'declined';
+    readonly error: unknown;
+} | {
+    readonly kind: 'handled';
 };
 /**
  * Coordinates a serving subscriber's gap-free transition from replica-local
@@ -42,7 +66,7 @@ export declare class SQLiteChangeLogCatchup implements Disposable {
      * the background so subscribe() does not wait for SQLite to reach the
      * required head.
      */
-    catchup(subscriber: Subscriber, captureRequiredHead: () => string | Promise<string>): Promise<void>;
+    catchup(subscriber: Subscriber, captureRequiredHead: () => string | Promise<string>, request?: SQLiteChangeLogCatchupRequest): Promise<SQLiteChangeLogCatchupRegistration>;
     /**
      * Notes that the change-log writer has committed `watermark`, i.e. that the
      * log's head has advanced to it. The writer runs in this process, so this is

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-catchup.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,KAAK,EAAC,MAAM,iCAAiC,CAAC;AAEtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE5D,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,+BAA+B,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,WAAW,4BAA4B;IAC3C,IAAI,CAAC,aAAa,EAAE,MAAM,GAAG,WAAW,CAAC;IACzC,IAAI,CACF,aAAa,EAAE,MAAM,EACrB,gBAAgB,EAAE,MAAM,EACxB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,WAAW,GACnB,aAAa,CAAC,SAAS,iBAAiB,EAAE,CAAC,CAAC;IAC/C,KAAK,IAAI,IAAI,CAAC;CACf;AAED;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IAC1C,oBAAoB,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CACxD;AAQD,MAAM,MAAM,6BAA6B,GAAG;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD,KAAK,CAAC,EAAE,OAAO,KAAK,GAAG,SAAS,CAAC;IACjC,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;CAClC,CAAC;AAMF;;;;;;;;GAQG;AACH,qBAAa,sBAAuB,YAAW,UAAU;;gBAiCrD,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,SAAS,EACpB,MAAM,EAAE,4BAA4B,EACpC,IAAI,EAAE,6BAA6B;IAcrC;;;;OAIG;IACG,OAAO,CACX,UAAU,EAAE,UAAU,EACtB,mBAAmB,EAAE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAClD,OAAO,CAAC,IAAI,CAAC;IA8BhB;;;;;;;;OAQG;IACH,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAS1C,MAAM,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IAMpC,KAAK,IAAI,IAAI;IAab,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAoRzB;AAID;;;;GAIG;AACH,qBAAa,2BAA4B,SAAQ,KAAK;CAAG;AAEzD,uEAAuE;AACvE,qBAAa,kCAAmC,SAAQ,2BAA2B;IACjF,QAAQ,CAAC,IAAI,wCAAwC;CACtD;AAED,iEAAiE;AACjE,qBAAa,kCAAmC,SAAQ,2BAA2B;IACjF,QAAQ,CAAC,IAAI,wCAAwC;CACtD"}
+{"version":3,"file":"sqlite-change-log-catchup.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,KAAK,EAAC,MAAM,iCAAiC,CAAC;AAMtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE5D,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,+BAA+B,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,WAAW,4BAA4B;IAC3C,IAAI,CAAC,aAAa,EAAE,MAAM,GAAG,WAAW,CAAC;IACzC,IAAI,CACF,aAAa,EAAE,MAAM,EACrB,gBAAgB,EAAE,MAAM,EACxB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,WAAW,GACnB,aAAa,CAAC,SAAS,iBAAiB,EAAE,CAAC,CAAC;IAC/C,KAAK,IAAI,IAAI,CAAC;CACf;AAED;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IAC1C,oBAAoB,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CACxD;AAQD,MAAM,MAAM,6BAA6B,GAAG;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD,KAAK,CAAC,EAAE,OAAO,KAAK,GAAG,SAAS,CAAC;IACjC,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC,+EAA+E;IAC/E,SAAS,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,6BAA6B,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;CAC5E,CAAC;AAEF,MAAM,MAAM,6BAA6B,GAAG;IAC1C;;;;;;OAMG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACxC,CAAC;AAEF,MAAM,MAAM,6BAA6B,GAAG,iBAAiB,GAAG,cAAc,CAAC;AAE/E,MAAM,MAAM,kCAAkC,GAC1C;IAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAA;CAAC,GAC7B;IAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;CAAC,GAG3D;IAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;CAAC,GAEpD;IAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAA;CAAC,CAAC;AAM/B;;;;;;;;GAQG;AACH,qBAAa,sBAAuB,YAAW,UAAU;;gBA4ErD,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,SAAS,EACpB,MAAM,EAAE,4BAA4B,EACpC,IAAI,EAAE,6BAA6B;IAerC;;;;OAIG;IACG,OAAO,CACX,UAAU,EAAE,UAAU,EACtB,mBAAmB,EAAE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,EACnD,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,kCAAkC,CAAC;IAiE9C;;;;;;;;OAQG;IACH,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAS1C,MAAM,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IAMpC,KAAK,IAAI,IAAI;IAab,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAgUzB;AAID;;;;GAIG;AACH,qBAAa,2BAA4B,SAAQ,KAAK;CAAG;AAEzD,uEAAuE;AACvE,qBAAa,kCAAmC,SAAQ,2BAA2B;IACjF,QAAQ,CAAC,IAAI,wCAAwC;CACtD;AAED,iEAAiE;AACjE,qBAAa,kCAAmC,SAAQ,2BAA2B;IACjF,QAAQ,CAAC,IAAI,wCAAwC;CACtD"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js

@@ -1,6 +1,6 @@
 import { AbortError } from "../../../../shared/src/abort-error.js";
 import { sleep } from "../../../../shared/src/sleep.js";
-import { getOrCreateCounter } from "../../observability/metrics.js";
+import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateValueHistogram } from "../../observability/metrics.js";
 import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts
 var DEFAULT_BARRIER_POLL_INTERVAL_MS = 1e3;
@@ -24,9 +24,32 @@ var SQLiteChangeLogCatchup = class {
 	#cleanupGuard;
 	#sleep;
 	#now;
+	#onFailure;
 	#barrierTimeouts = getOrCreateCounter("replication", "sqlite_change_log.barrier_timeouts", "SQLite change-log catchups that timed out waiting for the required head.");
 	#barrierBacklogOverflows = getOrCreateCounter("replication", "sqlite_change_log.barrier_backlog_overflows", "SQLite change-log catchups abandoned because the subscriber backlog reached its high water mark while waiting for the required head.");
 	#barrierWakeups = getOrCreateCounter("replication", "sqlite_change_log.barrier_wakeups", "SQLite catchup barrier waits, labeled by what ended the wait. A population dominated by \"poll\" means the change-log writer's commit notification is not reaching the barrier.");
+	#catchupResults = getOrCreateCounter("replication", "sqlite_change_log.catchup_results", "SQLite catchup subscriptions by outcome.");
+	#catchupRows = getOrCreateCounter("replication", "sqlite_change_log.catchup_rows", "Rows served from the SQLite change log during catchup.");
+	#catchupBytes = getOrCreateCounter("replication", "sqlite_change_log.catchup_bytes", {
+		description: "Bytes served from the SQLite change log during catchup.",
+		unit: "By"
+	});
+	#catchupDuration = getOrCreateLatencyHistogram("replication", "sqlite_change_log.catchup_duration", "Duration of a SQLite change-log catchup.");
+	#barrierWaitDuration = getOrCreateLatencyHistogram("replication", "sqlite_change_log.barrier_wait_duration", "Time SQLite catchup waits for its pinned required head.");
+	#catchupBacklogPeak = getOrCreateValueHistogram("replication", "sqlite_change_log.catchup_backlog_peak_bytes", {
+		description: "Peak live backlog bytes buffered while a subscriber catches up from SQLite.",
+		unit: "By",
+		bucketBoundaries: [
+			0,
+			1024,
+			16384,
+			65536,
+			262144,
+			1048576,
+			4194304,
+			16777216
+		]
+	});
 	#catchups = /* @__PURE__ */ new Map();
 	#commitWaiters = /* @__PURE__ */ new Set();
 	#closed = false;
@@ -40,35 +63,58 @@ var SQLiteChangeLogCatchup = class {
 		this.#cleanupGuard = opts.cleanupGuard ?? NOOP_CLEANUP_GUARD;
 		this.#sleep = opts.sleep ?? sleep;
 		this.#now = opts.now ?? Date.now;
+		this.#onFailure = opts.onFailure;
 	}
 	/**
 	* Registers the subscriber before resolving. Catchup itself continues in
 	* the background so subscribe() does not wait for SQLite to reach the
 	* required head.
 	*/
-	async catchup(subscriber, captureRequiredHead) {
+	async catchup(subscriber, captureRequiredHead, request = {}) {
 		if (this.#closed) {
 			subscriber.fail(new AbortError("SQLite change-log catchup is closed"));
-			return;
+			return { kind: "handled" };
 		}
 		const abort = new AbortController();
 		this.#catchups.set(subscriber, abort);
 		let requiredHead;
+		let uncoveredMinWatermark;
+		let committed = false;
 		try {
 			await this.#cleanupGuard.runWhilePurgeBlocked(() => {
 				this.#throwIfAborted(abort.signal);
+				const plan = this.#reader.plan(subscriber.watermark);
+				if (plan.kind === "too-old") {
+					uncoveredMinWatermark = plan.minWatermark;
+					return;
+				}
 				requiredHead = captureRequiredHead();
 				this.#forwarder.add(subscriber);
+				committed = true;
 			});
 		} catch (error) {
 			this.#catchups.delete(subscriber);
-			if (!abort.signal.aborted) {
-				this.#lc.error?.(`error while registering SQLite catchup subscriber ${subscriber.id}`, error);
-				subscriber.fail(error);
+			if (abort.signal.aborted) return { kind: "handled" };
+			if (!committed) {
+				this.#onFailure?.("reader-error");
+				return {
+					kind: "declined",
+					error
+				};
 			}
-			return;
+			this.#lc.error?.(`error while registering SQLite catchup subscriber ${subscriber.id}`, error);
+			subscriber.fail(error);
+			return { kind: "handled" };
+		}
+		if (uncoveredMinWatermark !== void 0) {
+			this.#catchups.delete(subscriber);
+			return {
+				kind: "uncovered",
+				minWatermark: uncoveredMinWatermark
+			};
 		}
-		this.#run(subscriber, requiredHead, abort);
+		this.#run(subscriber, requiredHead, abort, request.logWarm);
+		return { kind: "registered" };
 	}
 	/**
 	* Notes that the change-log writer has committed `watermark`, i.e. that the
@@ -103,23 +149,34 @@ var SQLiteChangeLogCatchup = class {
 	[Symbol.dispose]() {
 		this.close();
 	}
-	async #run(subscriber, requiredHead, abort) {
+	async #run(subscriber, requiredHead, abort, logWarm) {
 		const { signal } = abort;
+		const catchupStart = this.#now();
+		let outcome = "aborted";
+		let backlogPeakBytes = 0;
 		try {
 			const requiredHeadStart = this.#now();
 			const required = await this.#awaitRequiredHead(requiredHead, signal);
 			const requiredHeadWaitMs = this.#now() - requiredHeadStart;
 			if (requiredHeadWaitMs > this.#barrierTimeoutMs) this.#lc.info?.(`waited ${requiredHeadWaitMs} ms for the forwarded transaction to commit before starting the SQLite barrier for ${subscriber.id}`);
 			const deadline = this.#now() + this.#barrierTimeoutMs;
-			const plan = await this.#waitForPlan(subscriber, required, deadline, signal);
+			const barrierStart = this.#now();
+			let plan;
+			try {
+				plan = await this.#waitForPlan(subscriber, required, deadline, signal);
+			} finally {
+				this.#barrierWaitDuration.recordMs(this.#now() - barrierStart);
+			}
 			this.#throwIfAborted(signal);
 			if (plan.kind === "too-old") {
+				outcome = "too-old";
 				const message = `earliest supported watermark is ${plan.minWatermark} (requested ${subscriber.watermark})`;
 				this.#lc.warn?.(`rejecting subscriber at watermark ${subscriber.watermark} (earliest watermark: ${plan.minWatermark})`);
 				subscriber.close(2, message);
 				return;
 			}
 			let count = 0;
+			let bytes = 0;
 			const start = this.#now();
 			if (plan.kind === "range") {
 				let lastBatchConsumed;
@@ -132,24 +189,50 @@ var SQLiteChangeLogCatchup = class {
 					for (const change of changes) {
 						lastBatchConsumed = subscriber.catchup(change);
 						count++;
+						bytes += Buffer.byteLength(change[2]);
 					}
+					backlogPeakBytes = Math.max(backlogPeakBytes, subscriber.getStats().backlogBytes);
 				}
 				await lastBatchConsumed;
-			} else this.#lc.warn?.(`subscriber ${subscriber.id} at watermark ${subscriber.watermark} is ahead of the SQLite change-log head ${plan.headWatermark}; waiting for the replica to catch up`);
+				outcome = "range";
+			} else {
+				outcome = "ahead";
+				this.#lc.warn?.(`subscriber ${subscriber.id} at watermark ${subscriber.watermark} is ahead of the SQLite change-log head ${plan.headWatermark}; waiting for the replica to catch up`);
+			}
 			this.#throwIfAborted(signal);
 			this.#lc.info?.(`caught up ${subscriber.id} from SQLite with ${count} changes (${this.#now() - start} ms)`);
+			const attributes = {
+				classification: outcome,
+				log_warm: logWarm ?? "unknown"
+			};
+			this.#catchupRows.add(count, attributes);
+			this.#catchupBytes.add(bytes, attributes);
 			subscriber.setCaughtUp();
 		} catch (error) {
 			if (signal.aborted) return;
 			if (error instanceof SQLiteChangeLogBarrierError) {
-				if (error instanceof SQLiteChangeLogBarrierTimeoutError) this.#barrierTimeouts.add(1);
+				if (error instanceof SQLiteChangeLogBarrierTimeoutError) {
+					this.#barrierTimeouts.add(1);
+					outcome = "barrier-timeout";
+					this.#onFailure?.("barrier-timeout");
+				} else outcome = "barrier-backlog";
 				this.#lc.warn?.(`ending subscription for ${subscriber.id} to retry SQLite catchup`, error);
 				subscriber.close();
 				return;
 			}
+			outcome = "reader-error";
+			this.#onFailure?.("reader-error");
 			this.#lc.error?.(`error while catching up subscriber ${subscriber.id} from SQLite`, error);
 			subscriber.fail(error);
 		} finally {
+			backlogPeakBytes = Math.max(backlogPeakBytes, subscriber.getStats().backlogBytes);
+			const attributes = {
+				classification: outcome,
+				log_warm: logWarm ?? "unknown"
+			};
+			this.#catchupResults.add(1, attributes);
+			this.#catchupDuration.recordMs(this.#now() - catchupStart, attributes);
+			this.#catchupBacklogPeak.record(backlogPeakBytes, attributes);
 			if (this.#catchups.get(subscriber) === abort) this.#catchups.delete(subscriber);
 		}
 	}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-catchup.js","names":["#lc","#forwarder","#reader","#batchSize","#barrierTimeoutMs","#barrierPollIntervalMs","#cleanupGuard","#sleep","#now","#barrierTimeouts","#barrierBacklogOverflows","#barrierWakeups","#catchups","#commitWaiters","#closed","#throwIfAborted","#run","#awaitRequiredHead","#waitForPlan","#awaitChangeLogProgress"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {Forwarder} from './forwarder.ts';\nimport type {CatchupPlan} from './sqlite-change-log-reader.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport interface SQLiteChangeLogCatchupReader {\n  plan(fromWatermark: string): CatchupPlan;\n  read(\n    fromWatermark: string,\n    throughWatermark: string,\n    batchSize: number,\n    signal?: AbortSignal,\n  ): AsyncIterable<readonly WatermarkedChange[]>;\n  close(): void;\n}\n\n/**\n * Slice 9 replaces this no-op implementation with a guard that waits for an\n * in-flight purge and blocks new purge dispatch while the subscriber's ACK is\n * made visible to the Forwarder.\n */\nexport interface SQLiteChangeLogCleanupGuard {\n  runWhilePurgeBlocked<T>(register: () => T): Promise<T>;\n}\n\n// How long the barrier waits for the change-log writer's commit notification\n// before re-reading the log itself. The notification normally arrives first, so\n// this is a backstop for the windows in which none is coming: the writer is\n// disabled, or it is between stream connections.\nconst DEFAULT_BARRIER_POLL_INTERVAL_MS = 1000;\n\nexport type SQLiteChangeLogCatchupOptions = {\n  batchSize: number;\n  barrierTimeoutMs: number;\n  barrierPollIntervalMs?: number | undefined;\n  cleanupGuard?: SQLiteChangeLogCleanupGuard | undefined;\n  sleep?: typeof sleep | undefined;\n  now?: (() => number) | undefined;\n};\n\nconst NOOP_CLEANUP_GUARD: SQLiteChangeLogCleanupGuard = {\n  runWhilePurgeBlocked: register => Promise.resolve(register()),\n};\n\n/**\n * Coordinates a serving subscriber's gap-free transition from replica-local\n * SQLite catchup to the Forwarder's live stream.\n *\n * Registration and required-head capture happen in one synchronous callback.\n * The subscriber therefore either sees a transaction in SQLite catchup or in\n * its live backlog (duplicates across that boundary are filtered by\n * Subscriber), never in neither place.\n */\nexport class SQLiteChangeLogCatchup implements Disposable {\n  readonly #lc: LogContext;\n  readonly #forwarder: Forwarder;\n  readonly #reader: SQLiteChangeLogCatchupReader;\n  readonly #batchSize: number;\n  readonly #barrierTimeoutMs: number;\n  readonly #barrierPollIntervalMs: number;\n  readonly #cleanupGuard: SQLiteChangeLogCleanupGuard;\n  readonly #sleep: typeof sleep;\n  readonly #now: () => number;\n  readonly #barrierTimeouts = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_timeouts',\n    'SQLite change-log catchups that timed out waiting for the required head.',\n  );\n  readonly #barrierBacklogOverflows = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_backlog_overflows',\n    'SQLite change-log catchups abandoned because the subscriber backlog ' +\n      'reached its high water mark while waiting for the required head.',\n  );\n  readonly #barrierWakeups = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_wakeups',\n    'SQLite catchup barrier waits, labeled by what ended the wait. A ' +\n      'population dominated by \"poll\" means the change-log writer\\'s commit ' +\n      'notification is not reaching the barrier.',\n  );\n  readonly #catchups = new Map<Subscriber, AbortController>();\n  readonly #commitWaiters = new Set<CommitWaiter>();\n  #closed = false;\n\n  constructor(\n    lc: LogContext,\n    forwarder: Forwarder,\n    reader: SQLiteChangeLogCatchupReader,\n    opts: SQLiteChangeLogCatchupOptions,\n  ) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-catchup');\n    this.#forwarder = forwarder;\n    this.#reader = reader;\n    this.#batchSize = opts.batchSize;\n    this.#barrierTimeoutMs = opts.barrierTimeoutMs;\n    this.#barrierPollIntervalMs =\n      opts.barrierPollIntervalMs ?? DEFAULT_BARRIER_POLL_INTERVAL_MS;\n    this.#cleanupGuard = opts.cleanupGuard ?? NOOP_CLEANUP_GUARD;\n    this.#sleep = opts.sleep ?? sleep;\n    this.#now = opts.now ?? Date.now;\n  }\n\n  /**\n   * Registers the subscriber before resolving. Catchup itself continues in\n   * the background so subscribe() does not wait for SQLite to reach the\n   * required head.\n   */\n  async catchup(\n    subscriber: Subscriber,\n    captureRequiredHead: () => string | Promise<string>,\n  ): Promise<void> {\n    if (this.#closed) {\n      subscriber.fail(new AbortError('SQLite change-log catchup is closed'));\n      return;\n    }\n\n    const abort = new AbortController();\n    this.#catchups.set(subscriber, abort);\n    let requiredHead: string | Promise<string> | undefined;\n    try {\n      await this.#cleanupGuard.runWhilePurgeBlocked(() => {\n        this.#throwIfAborted(abort.signal);\n        requiredHead = captureRequiredHead();\n        this.#forwarder.add(subscriber);\n      });\n    } catch (error) {\n      this.#catchups.delete(subscriber);\n      if (!abort.signal.aborted) {\n        this.#lc.error?.(\n          `error while registering SQLite catchup subscriber ${subscriber.id}`,\n          error,\n        );\n        subscriber.fail(error);\n      }\n      return;\n    }\n\n    void this.#run(subscriber, requiredHead as string | Promise<string>, abort);\n  }\n\n  /**\n   * Notes that the change-log writer has committed `watermark`, i.e. that the\n   * log's head has advanced to it. The writer runs in this process, so this is\n   * the event itself rather than a subscriber's ACK of it.\n   *\n   * It is a wakeup, not an authority: `plan()` still decides what can be read.\n   * A notification that never arrives costs the barrier a poll interval, and one\n   * that arrives early costs an extra `plan()` call.\n   */\n  onChangeLogCommit(watermark: string): void {\n    for (const waiter of this.#commitWaiters) {\n      if (watermark >= waiter.watermark) {\n        this.#commitWaiters.delete(waiter);\n        waiter.resolve();\n      }\n    }\n  }\n\n  remove(subscriber: Subscriber): void {\n    this.#catchups.get(subscriber)?.abort();\n    this.#catchups.delete(subscriber);\n    this.#forwarder.remove(subscriber);\n  }\n\n  close(): void {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    for (const [subscriber, abort] of this.#catchups) {\n      abort.abort();\n      this.#forwarder.remove(subscriber);\n    }\n    this.#catchups.clear();\n    this.#reader.close();\n  }\n\n  [Symbol.dispose](): void {\n    this.close();\n  }\n\n  async #run(\n    subscriber: Subscriber,\n    requiredHead: string | Promise<string>,\n    abort: AbortController,\n  ): Promise<void> {\n    const {signal} = abort;\n    try {\n      const requiredHeadStart = this.#now();\n      const required = await this.#awaitRequiredHead(requiredHead, signal);\n      const requiredHeadWaitMs = this.#now() - requiredHeadStart;\n      if (requiredHeadWaitMs > this.#barrierTimeoutMs) {\n        this.#lc.info?.(\n          `waited ${requiredHeadWaitMs} ms for the forwarded transaction to ` +\n            `commit before starting the SQLite barrier for ${subscriber.id}`,\n        );\n      }\n      // The barrier bounds replica lag, not upstream transaction duration, so\n      // its deadline starts once the required head is known. Sharing one\n      // deadline with the wait above would fail subscribers that registered\n      // during a transaction longer than barrierTimeoutMs, however current\n      // the replica is.\n      const deadline = this.#now() + this.#barrierTimeoutMs;\n      const plan = await this.#waitForPlan(\n        subscriber,\n        required,\n        deadline,\n        signal,\n      );\n      this.#throwIfAborted(signal);\n\n      if (plan.kind === 'too-old') {\n        const message =\n          `earliest supported watermark is ${plan.minWatermark} ` +\n          `(requested ${subscriber.watermark})`;\n        this.#lc.warn?.(\n          `rejecting subscriber at watermark ${subscriber.watermark} ` +\n            `(earliest watermark: ${plan.minWatermark})`,\n        );\n        subscriber.close(ErrorType.WatermarkTooOld, message);\n        return;\n      }\n\n      let count = 0;\n      const start = this.#now();\n      if (plan.kind === 'range') {\n        let lastBatchConsumed: Promise<unknown> | undefined;\n        for await (const changes of this.#reader.read(\n          subscriber.watermark,\n          plan.headWatermark,\n          this.#batchSize,\n          signal,\n        )) {\n          const waitStart = this.#now();\n          await lastBatchConsumed;\n          const elapsed = this.#now() - waitStart;\n          if (lastBatchConsumed) {\n            this.#lc[elapsed > 100 ? 'info' : 'debug']?.(\n              `waited ${elapsed.toFixed(3)} ms for ${subscriber.id} to consume ` +\n                `the previous SQLite catchup batch`,\n            );\n          }\n          this.#throwIfAborted(signal);\n          for (const change of changes) {\n            lastBatchConsumed = subscriber.catchup(change);\n            count++;\n          }\n        }\n        await lastBatchConsumed;\n      } else {\n        this.#lc.warn?.(\n          `subscriber ${subscriber.id} at watermark ${subscriber.watermark} ` +\n            `is ahead of the SQLite change-log head ${plan.headWatermark}; ` +\n            `waiting for the replica to catch up`,\n        );\n      }\n\n      this.#throwIfAborted(signal);\n      this.#lc.info?.(\n        `caught up ${subscriber.id} from SQLite with ${count} changes ` +\n          `(${this.#now() - start} ms)`,\n      );\n      // Keep buffering live sends until the asynchronous backlog drain has\n      // established its ordering boundary.\n      void subscriber.setCaughtUp();\n    } catch (error) {\n      if (signal.aborted) {\n        return;\n      }\n      if (error instanceof SQLiteChangeLogBarrierError) {\n        if (error instanceof SQLiteChangeLogBarrierTimeoutError) {\n          this.#barrierTimeouts.add(1);\n        }\n        // Giving up on the barrier means the replica has not caught up *yet*,\n        // not that it cannot serve this subscriber. End the subscription\n        // cleanly so the client reconnects and retries: IncrementalSyncer\n        // treats any ['error', ...] message as terminal and restores a fresh\n        // replica from litestream, whereas a clean end backs off and\n        // re-subscribes. If retries keep failing until the change log is purged\n        // past the subscriber's watermark, plan() returns 'too-old', which is\n        // terminal by design.\n        this.#lc.warn?.(\n          `ending subscription for ${subscriber.id} to retry SQLite catchup`,\n          error,\n        );\n        subscriber.close();\n        return;\n      }\n      this.#lc.error?.(\n        `error while catching up subscriber ${subscriber.id} from SQLite`,\n        error,\n      );\n      subscriber.fail(error);\n    } finally {\n      if (this.#catchups.get(subscriber) === abort) {\n        this.#catchups.delete(subscriber);\n      }\n    }\n  }\n\n  /**\n   * Waits for the transaction that was in flight at registration to finish.\n   *\n   * This wait is bounded by the upstream transaction rather than by\n   * `barrierTimeoutMs`: the completion settles on both commit and rollback,\n   * and an interrupted change stream forwards a synthetic rollback, so it\n   * always settles. Aborting -- the subscriber disconnecting or the service\n   * shutting down -- is what releases it early.\n   */\n  async #awaitRequiredHead(\n    requiredHead: string | Promise<string>,\n    signal: AbortSignal,\n  ): Promise<string> {\n    if (typeof requiredHead === 'string') {\n      return requiredHead;\n    }\n    this.#throwIfAborted(signal);\n    let onAbort: (() => void) | undefined;\n    try {\n      return await Promise.race([\n        requiredHead,\n        new Promise<never>((_, reject) => {\n          onAbort = () =>\n            reject(new AbortError('SQLite change-log catchup aborted'));\n          signal.addEventListener('abort', onAbort, {once: true});\n        }),\n      ]);\n    } finally {\n      if (onAbort) {\n        signal.removeEventListener('abort', onAbort);\n      }\n    }\n  }\n\n  async #waitForPlan(\n    subscriber: Subscriber,\n    requiredHead: string,\n    deadline: number,\n    signal: AbortSignal,\n  ): Promise<Exclude<CatchupPlan, {kind: 'not-ready'}>> {\n    // Race backlog growth once around the whole polling loop. Attaching the\n    // same unresolved promise to every per-poll race would retain one reaction\n    // per iteration until the subscriber's backlog eventually filled.\n    const backlogFull = subscriber.whenBacklogFull();\n    const barrier = new AbortController();\n    const abortBarrier = () => barrier.abort();\n    signal.addEventListener('abort', abortBarrier, {once: true});\n    if (signal.aborted) {\n      barrier.abort();\n    }\n\n    let observedHead = subscriber.watermark;\n    try {\n      const planReady = (async () => {\n        while (true) {\n          this.#throwIfAborted(barrier.signal);\n          const plan = this.#reader.plan(subscriber.watermark);\n          // A not-ready log has no head to compare, so it can never satisfy\n          // the required head: keep waiting for the writer to create and\n          // reconcile it, and let the barrier deadline end the subscription\n          // cleanly for a retry. Selection declines a not-ready log before a\n          // subscriber gets here, and neither purging nor reconciliation can\n          // empty a log that had content, so this is a belt-and-braces path.\n          if (plan.kind !== 'not-ready') {\n            observedHead = plan.headWatermark;\n            if (plan.headWatermark >= requiredHead) {\n              return plan;\n            }\n          }\n          const remaining = deadline - this.#now();\n          if (remaining <= 0) {\n            throw new SQLiteChangeLogBarrierTimeoutError(\n              `timed out waiting for SQLite head ${observedHead} to ` +\n                `reach required head ${requiredHead}`,\n            );\n          }\n          await this.#awaitChangeLogProgress(\n            requiredHead,\n            remaining,\n            barrier.signal,\n          );\n        }\n      })();\n\n      const backlogOverflow = backlogFull.promise.then(() => {\n        this.#throwIfAborted(signal);\n        // close() also resolves backlog waiters so none are stranded. The\n        // backlog is cleared first, which distinguishes close from overflow.\n        if (!subscriber.backlogFull) {\n          return new Promise<never>(() => {});\n        }\n        this.#barrierWakeups.add(1, {'wakeup.source': 'backlog'});\n        // Bytes, not time, are what the barrier actually costs: past the high\n        // water mark this subscriber's send() no longer resolves, so it holds\n        // up every flush and can stall the replica this barrier waits on.\n        this.#barrierBacklogOverflows.add(1);\n        throw new SQLiteChangeLogBarrierBacklogError(\n          `backlog for subscriber ${subscriber.id} reached its high water ` +\n            `mark while waiting for SQLite head ${observedHead} to ` +\n            `reach required head ${requiredHead}`,\n        );\n      });\n\n      return await Promise.race([planReady, backlogOverflow]);\n    } finally {\n      signal.removeEventListener('abort', abortBarrier);\n      barrier.abort();\n      backlogFull.cancel();\n    }\n  }\n\n  /**\n   * Waits for the change-log writer to commit `requiredHead`, or the poll\n   * interval as a backstop.\n   *\n   * No notification can be missed by registering after `plan()` was read: the\n   * notification follows the commit it reports, so if the writer had already\n   * committed `requiredHead`, `plan()` would have seen it.\n   */\n  async #awaitChangeLogProgress(\n    requiredHead: string,\n    remainingMs: number,\n    signal: AbortSignal,\n  ): Promise<void> {\n    const committed = resolver<void>();\n    const waiter = {watermark: requiredHead, resolve: committed.resolve};\n    this.#commitWaiters.add(waiter);\n    // Scopes the backstop timer to this wait so that a commit cancels it rather\n    // than leaving it to fire against a barrier that has already moved on.\n    const poll = new AbortController();\n    const abortPoll = () => poll.abort();\n    signal.addEventListener('abort', abortPoll, {once: true});\n    try {\n      const wokenBy = await Promise.race([\n        committed.promise.then(() => 'commit' as const),\n        this.#sleep(\n          Math.min(this.#barrierPollIntervalMs, remainingMs),\n          poll.signal,\n        ).then(() => 'poll' as const),\n      ]);\n      this.#barrierWakeups.add(1, {'wakeup.source': wokenBy});\n    } finally {\n      this.#commitWaiters.delete(waiter);\n      signal.removeEventListener('abort', abortPoll);\n      poll.abort();\n    }\n  }\n\n  #throwIfAborted(signal: AbortSignal): void {\n    if (this.#closed || signal.aborted) {\n      throw new AbortError('SQLite change-log catchup aborted');\n    }\n  }\n}\n\ntype CommitWaiter = {watermark: string; resolve: () => void};\n\n/**\n * A barrier that gave up on the replica reaching the required head. Both\n * reasons mean \"not yet\", not \"never\", so both end the subscription cleanly\n * for a retry rather than failing it.\n */\nexport class SQLiteChangeLogBarrierError extends Error {}\n\n/** The replica did not reach the required head before the deadline. */\nexport class SQLiteChangeLogBarrierTimeoutError extends SQLiteChangeLogBarrierError {\n  readonly name = 'SQLiteChangeLogBarrierTimeoutError';\n}\n\n/** Waiting any longer would have cost more than reconnecting. */\nexport class SQLiteChangeLogBarrierBacklogError extends SQLiteChangeLogBarrierError {\n  readonly name = 'SQLiteChangeLogBarrierBacklogError';\n}\n"],"mappings":";;;;;AAmCA,IAAM,mCAAmC;AAWzC,IAAM,qBAAkD,EACtD,uBAAsB,aAAY,QAAQ,QAAQ,SAAS,CAAC,EAC9D;;;;;;;;;;AAWA,IAAa,yBAAb,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,mBAA4B,mBAC1B,eACA,sCACA,0EACF;CACA,2BAAoC,mBAClC,eACA,+CACA,sIAEF;CACA,kBAA2B,mBACzB,eACA,qCACA,iLAGF;CACA,4BAAqB,IAAI,IAAiC;CAC1D,iCAA0B,IAAI,IAAkB;CAChD,UAAU;CAEV,YACE,IACA,WACA,QACA,MACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAKC,aAAa;EAClB,KAAKC,UAAU;EACf,KAAKC,aAAa,KAAK;EACvB,KAAKC,oBAAoB,KAAK;EAC9B,KAAKC,yBACH,KAAK,yBAAyB;EAChC,KAAKC,gBAAgB,KAAK,gBAAgB;EAC1C,KAAKC,SAAS,KAAK,SAAS;EAC5B,KAAKC,OAAO,KAAK,OAAO,KAAK;CAC/B;;;;;;CAOA,MAAM,QACJ,YACA,qBACe;EACf,IAAI,KAAKM,SAAS;GAChB,WAAW,KAAK,IAAI,WAAW,qCAAqC,CAAC;GACrE;EACF;EAEA,MAAM,QAAQ,IAAI,gBAAgB;EAClC,KAAKF,UAAU,IAAI,YAAY,KAAK;EACpC,IAAI;EACJ,IAAI;GACF,MAAM,KAAKN,cAAc,2BAA2B;IAClD,KAAKS,gBAAgB,MAAM,MAAM;IACjC,eAAe,oBAAoB;IACnC,KAAKd,WAAW,IAAI,UAAU;GAChC,CAAC;EACH,SAAS,OAAO;GACd,KAAKW,UAAU,OAAO,UAAU;GAChC,IAAI,CAAC,MAAM,OAAO,SAAS;IACzB,KAAKZ,IAAI,QACP,qDAAqD,WAAW,MAChE,KACF;IACA,WAAW,KAAK,KAAK;GACvB;GACA;EACF;EAEA,KAAUgB,KAAK,YAAY,cAA0C,KAAK;CAC5E;;;;;;;;;;CAWA,kBAAkB,WAAyB;EACzC,KAAK,MAAM,UAAU,KAAKH,gBACxB,IAAI,aAAa,OAAO,WAAW;GACjC,KAAKA,eAAe,OAAO,MAAM;GACjC,OAAO,QAAQ;EACjB;CAEJ;CAEA,OAAO,YAA8B;EACnC,KAAKD,UAAU,IAAI,UAAU,GAAG,MAAM;EACtC,KAAKA,UAAU,OAAO,UAAU;EAChC,KAAKX,WAAW,OAAO,UAAU;CACnC;CAEA,QAAc;EACZ,IAAI,KAAKa,SACP;EAEF,KAAKA,UAAU;EACf,KAAK,MAAM,CAAC,YAAY,UAAU,KAAKF,WAAW;GAChD,MAAM,MAAM;GACZ,KAAKX,WAAW,OAAO,UAAU;EACnC;EACA,KAAKW,UAAU,MAAM;EACrB,KAAKV,QAAQ,MAAM;CACrB;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,MAAM;CACb;CAEA,MAAMc,KACJ,YACA,cACA,OACe;EACf,MAAM,EAAC,WAAU;EACjB,IAAI;GACF,MAAM,oBAAoB,KAAKR,KAAK;GACpC,MAAM,WAAW,MAAM,KAAKS,mBAAmB,cAAc,MAAM;GACnE,MAAM,qBAAqB,KAAKT,KAAK,IAAI;GACzC,IAAI,qBAAqB,KAAKJ,mBAC5B,KAAKJ,IAAI,OACP,UAAU,mBAAmB,qFACsB,WAAW,IAChE;GAOF,MAAM,WAAW,KAAKQ,KAAK,IAAI,KAAKJ;GACpC,MAAM,OAAO,MAAM,KAAKc,aACtB,YACA,UACA,UACA,MACF;GACA,KAAKH,gBAAgB,MAAM;GAE3B,IAAI,KAAK,SAAS,WAAW;IAC3B,MAAM,UACJ,mCAAmC,KAAK,aAAa,cACvC,WAAW,UAAU;IACrC,KAAKf,IAAI,OACP,qCAAqC,WAAW,UAAU,wBAChC,KAAK,aAAa,EAC9C;IACA,WAAW,MAAM,GAA2B,OAAO;IACnD;GACF;GAEA,IAAI,QAAQ;GACZ,MAAM,QAAQ,KAAKQ,KAAK;GACxB,IAAI,KAAK,SAAS,SAAS;IACzB,IAAI;IACJ,WAAW,MAAM,WAAW,KAAKN,QAAQ,KACvC,WAAW,WACX,KAAK,eACL,KAAKC,YACL,MACF,GAAG;KACD,MAAM,YAAY,KAAKK,KAAK;KAC5B,MAAM;KACN,MAAM,UAAU,KAAKA,KAAK,IAAI;KAC9B,IAAI,mBACF,KAAKR,IAAI,UAAU,MAAM,SAAS,WAChC,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,WAAW,GAAG,8CAEvD;KAEF,KAAKe,gBAAgB,MAAM;KAC3B,KAAK,MAAM,UAAU,SAAS;MAC5B,oBAAoB,WAAW,QAAQ,MAAM;MAC7C;KACF;IACF;IACA,MAAM;GACR,OACE,KAAKf,IAAI,OACP,cAAc,WAAW,GAAG,gBAAgB,WAAW,UAAU,0CACrB,KAAK,cAAc,sCAEjE;GAGF,KAAKe,gBAAgB,MAAM;GAC3B,KAAKf,IAAI,OACP,aAAa,WAAW,GAAG,oBAAoB,MAAM,YAC/C,KAAKQ,KAAK,IAAI,MAAM,KAC5B;GAGA,WAAgB,YAAY;EAC9B,SAAS,OAAO;GACd,IAAI,OAAO,SACT;GAEF,IAAI,iBAAiB,6BAA6B;IAChD,IAAI,iBAAiB,oCACnB,KAAKC,iBAAiB,IAAI,CAAC;IAU7B,KAAKT,IAAI,OACP,2BAA2B,WAAW,GAAG,2BACzC,KACF;IACA,WAAW,MAAM;IACjB;GACF;GACA,KAAKA,IAAI,QACP,sCAAsC,WAAW,GAAG,eACpD,KACF;GACA,WAAW,KAAK,KAAK;EACvB,UAAU;GACR,IAAI,KAAKY,UAAU,IAAI,UAAU,MAAM,OACrC,KAAKA,UAAU,OAAO,UAAU;EAEpC;CACF;;;;;;;;;;CAWA,MAAMK,mBACJ,cACA,QACiB;EACjB,IAAI,OAAO,iBAAiB,UAC1B,OAAO;EAET,KAAKF,gBAAgB,MAAM;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,QAAQ,KAAK,CACxB,cACA,IAAI,SAAgB,GAAG,WAAW;IAChC,gBACE,OAAO,IAAI,WAAW,mCAAmC,CAAC;IAC5D,OAAO,iBAAiB,SAAS,SAAS,EAAC,MAAM,KAAI,CAAC;GACxD,CAAC,CACH,CAAC;EACH,UAAU;GACR,IAAI,SACF,OAAO,oBAAoB,SAAS,OAAO;EAE/C;CACF;CAEA,MAAMG,aACJ,YACA,cACA,UACA,QACoD;EAIpD,MAAM,cAAc,WAAW,gBAAgB;EAC/C,MAAM,UAAU,IAAI,gBAAgB;EACpC,MAAM,qBAAqB,QAAQ,MAAM;EACzC,OAAO,iBAAiB,SAAS,cAAc,EAAC,MAAM,KAAI,CAAC;EAC3D,IAAI,OAAO,SACT,QAAQ,MAAM;EAGhB,IAAI,eAAe,WAAW;EAC9B,IAAI;GACF,MAAM,aAAa,YAAY;IAC7B,OAAO,MAAM;KACX,KAAKH,gBAAgB,QAAQ,MAAM;KACnC,MAAM,OAAO,KAAKb,QAAQ,KAAK,WAAW,SAAS;KAOnD,IAAI,KAAK,SAAS,aAAa;MAC7B,eAAe,KAAK;MACpB,IAAI,KAAK,iBAAiB,cACxB,OAAO;KAEX;KACA,MAAM,YAAY,WAAW,KAAKM,KAAK;KACvC,IAAI,aAAa,GACf,MAAM,IAAI,mCACR,qCAAqC,aAAa,0BACzB,cAC3B;KAEF,MAAM,KAAKW,wBACT,cACA,WACA,QAAQ,MACV;IACF;GACF,GAAG;GAEH,MAAM,kBAAkB,YAAY,QAAQ,WAAW;IACrD,KAAKJ,gBAAgB,MAAM;IAG3B,IAAI,CAAC,WAAW,aACd,OAAO,IAAI,cAAqB,CAAC,CAAC;IAEpC,KAAKJ,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,UAAS,CAAC;IAIxD,KAAKD,yBAAyB,IAAI,CAAC;IACnC,MAAM,IAAI,mCACR,0BAA0B,WAAW,GAAG,6DACA,aAAa,0BAC5B,cAC3B;GACF,CAAC;GAED,OAAO,MAAM,QAAQ,KAAK,CAAC,WAAW,eAAe,CAAC;EACxD,UAAU;GACR,OAAO,oBAAoB,SAAS,YAAY;GAChD,QAAQ,MAAM;GACd,YAAY,OAAO;EACrB;CACF;;;;;;;;;CAUA,MAAMS,wBACJ,cACA,aACA,QACe;EACf,MAAM,YAAY,SAAe;EACjC,MAAM,SAAS;GAAC,WAAW;GAAc,SAAS,UAAU;EAAO;EACnE,KAAKN,eAAe,IAAI,MAAM;EAG9B,MAAM,OAAO,IAAI,gBAAgB;EACjC,MAAM,kBAAkB,KAAK,MAAM;EACnC,OAAO,iBAAiB,SAAS,WAAW,EAAC,MAAM,KAAI,CAAC;EACxD,IAAI;GACF,MAAM,UAAU,MAAM,QAAQ,KAAK,CACjC,UAAU,QAAQ,WAAW,QAAiB,GAC9C,KAAKN,OACH,KAAK,IAAI,KAAKF,wBAAwB,WAAW,GACjD,KAAK,MACP,EAAE,WAAW,MAAe,CAC9B,CAAC;GACD,KAAKM,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,QAAO,CAAC;EACxD,UAAU;GACR,KAAKE,eAAe,OAAO,MAAM;GACjC,OAAO,oBAAoB,SAAS,SAAS;GAC7C,KAAK,MAAM;EACb;CACF;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAKC,WAAW,OAAO,SACzB,MAAM,IAAI,WAAW,mCAAmC;CAE5D;AACF;;;;;;AASA,IAAa,8BAAb,cAAiD,MAAM,CAAC;;AAGxD,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB;;AAGA,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB"}
+{"version":3,"file":"sqlite-change-log-catchup.js","names":["#lc","#forwarder","#reader","#batchSize","#barrierTimeoutMs","#barrierPollIntervalMs","#cleanupGuard","#sleep","#now","#onFailure","#barrierTimeouts","#barrierBacklogOverflows","#barrierWakeups","#catchupResults","#catchupRows","#catchupBytes","#catchupDuration","#barrierWaitDuration","#catchupBacklogPeak","#catchups","#commitWaiters","#closed","#throwIfAborted","#run","#awaitRequiredHead","#waitForPlan","#awaitChangeLogProgress"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateLatencyHistogram,\n  getOrCreateValueHistogram,\n} from '../../observability/metrics.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {Forwarder} from './forwarder.ts';\nimport type {CatchupPlan} from './sqlite-change-log-reader.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport interface SQLiteChangeLogCatchupReader {\n  plan(fromWatermark: string): CatchupPlan;\n  read(\n    fromWatermark: string,\n    throughWatermark: string,\n    batchSize: number,\n    signal?: AbortSignal,\n  ): AsyncIterable<readonly WatermarkedChange[]>;\n  close(): void;\n}\n\n/**\n * Slice 9 replaces this no-op implementation with a guard that waits for an\n * in-flight purge and blocks new purge dispatch while the subscriber's ACK is\n * made visible to the Forwarder.\n */\nexport interface SQLiteChangeLogCleanupGuard {\n  runWhilePurgeBlocked<T>(register: () => T): Promise<T>;\n}\n\n// How long the barrier waits for the change-log writer's commit notification\n// before re-reading the log itself. The notification normally arrives first, so\n// this is a backstop for the windows in which none is coming: the writer is\n// disabled, or it is between stream connections.\nconst DEFAULT_BARRIER_POLL_INTERVAL_MS = 1000;\n\nexport type SQLiteChangeLogCatchupOptions = {\n  batchSize: number;\n  barrierTimeoutMs: number;\n  barrierPollIntervalMs?: number | undefined;\n  cleanupGuard?: SQLiteChangeLogCleanupGuard | undefined;\n  sleep?: typeof sleep | undefined;\n  now?: (() => number) | undefined;\n  /** Opens slice 11's process-local breaker after registration has committed. */\n  onFailure?: ((failure: SQLiteChangeLogCatchupFailure) => void) | undefined;\n};\n\nexport type SQLiteChangeLogCatchupRequest = {\n  /**\n   * Whether the routing warm-up gate classified the log as warm *for this\n   * subscriber*. The coordinator is opened once and reused, so this cannot be\n   * a property of it: `coldReadPercent` serves some subscribers from a log\n   * that is still inside its warm-up window, and the catchup metrics have to\n   * separate those from steady-state serving.\n   */\n  readonly logWarm?: boolean | undefined;\n};\n\nexport type SQLiteChangeLogCatchupFailure = 'barrier-timeout' | 'reader-error';\n\nexport type SQLiteChangeLogCatchupRegistration =\n  | {readonly kind: 'registered'}\n  | {readonly kind: 'uncovered'; readonly minWatermark: string}\n  // Registration failed before the subscriber was committed to SQLite, so PG\n  // catchup is still available and the caller falls back to it.\n  | {readonly kind: 'declined'; readonly error: unknown}\n  // The coordinator closed or failed the subscriber itself.\n  | {readonly kind: 'handled'};\n\nconst NOOP_CLEANUP_GUARD: SQLiteChangeLogCleanupGuard = {\n  runWhilePurgeBlocked: register => Promise.resolve(register()),\n};\n\n/**\n * Coordinates a serving subscriber's gap-free transition from replica-local\n * SQLite catchup to the Forwarder's live stream.\n *\n * Registration and required-head capture happen in one synchronous callback.\n * The subscriber therefore either sees a transaction in SQLite catchup or in\n * its live backlog (duplicates across that boundary are filtered by\n * Subscriber), never in neither place.\n */\nexport class SQLiteChangeLogCatchup implements Disposable {\n  readonly #lc: LogContext;\n  readonly #forwarder: Forwarder;\n  readonly #reader: SQLiteChangeLogCatchupReader;\n  readonly #batchSize: number;\n  readonly #barrierTimeoutMs: number;\n  readonly #barrierPollIntervalMs: number;\n  readonly #cleanupGuard: SQLiteChangeLogCleanupGuard;\n  readonly #sleep: typeof sleep;\n  readonly #now: () => number;\n  readonly #onFailure:\n    | ((failure: SQLiteChangeLogCatchupFailure) => void)\n    | undefined;\n  readonly #barrierTimeouts = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_timeouts',\n    'SQLite change-log catchups that timed out waiting for the required head.',\n  );\n  readonly #barrierBacklogOverflows = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_backlog_overflows',\n    'SQLite change-log catchups abandoned because the subscriber backlog ' +\n      'reached its high water mark while waiting for the required head.',\n  );\n  readonly #barrierWakeups = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_wakeups',\n    'SQLite catchup barrier waits, labeled by what ended the wait. A ' +\n      'population dominated by \"poll\" means the change-log writer\\'s commit ' +\n      'notification is not reaching the barrier.',\n  );\n  readonly #catchupResults = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.catchup_results',\n    'SQLite catchup subscriptions by outcome.',\n  );\n  readonly #catchupRows = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.catchup_rows',\n    'Rows served from the SQLite change log during catchup.',\n  );\n  readonly #catchupBytes = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.catchup_bytes',\n    {\n      description: 'Bytes served from the SQLite change log during catchup.',\n      unit: 'By',\n    },\n  );\n  readonly #catchupDuration = getOrCreateLatencyHistogram(\n    'replication',\n    'sqlite_change_log.catchup_duration',\n    'Duration of a SQLite change-log catchup.',\n  );\n  readonly #barrierWaitDuration = getOrCreateLatencyHistogram(\n    'replication',\n    'sqlite_change_log.barrier_wait_duration',\n    'Time SQLite catchup waits for its pinned required head.',\n  );\n  readonly #catchupBacklogPeak = getOrCreateValueHistogram(\n    'replication',\n    'sqlite_change_log.catchup_backlog_peak_bytes',\n    {\n      description:\n        'Peak live backlog bytes buffered while a subscriber catches up from SQLite.',\n      unit: 'By',\n      bucketBoundaries: [\n        0, 1024, 16_384, 65_536, 262_144, 1_048_576, 4_194_304, 16_777_216,\n      ],\n    },\n  );\n  readonly #catchups = new Map<Subscriber, AbortController>();\n  readonly #commitWaiters = new Set<CommitWaiter>();\n  #closed = false;\n\n  constructor(\n    lc: LogContext,\n    forwarder: Forwarder,\n    reader: SQLiteChangeLogCatchupReader,\n    opts: SQLiteChangeLogCatchupOptions,\n  ) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-catchup');\n    this.#forwarder = forwarder;\n    this.#reader = reader;\n    this.#batchSize = opts.batchSize;\n    this.#barrierTimeoutMs = opts.barrierTimeoutMs;\n    this.#barrierPollIntervalMs =\n      opts.barrierPollIntervalMs ?? DEFAULT_BARRIER_POLL_INTERVAL_MS;\n    this.#cleanupGuard = opts.cleanupGuard ?? NOOP_CLEANUP_GUARD;\n    this.#sleep = opts.sleep ?? sleep;\n    this.#now = opts.now ?? Date.now;\n    this.#onFailure = opts.onFailure;\n  }\n\n  /**\n   * Registers the subscriber before resolving. Catchup itself continues in\n   * the background so subscribe() does not wait for SQLite to reach the\n   * required head.\n   */\n  async catchup(\n    subscriber: Subscriber,\n    captureRequiredHead: () => string | Promise<string>,\n    request: SQLiteChangeLogCatchupRequest = {},\n  ): Promise<SQLiteChangeLogCatchupRegistration> {\n    if (this.#closed) {\n      subscriber.fail(new AbortError('SQLite change-log catchup is closed'));\n      return {kind: 'handled'};\n    }\n\n    const abort = new AbortController();\n    this.#catchups.set(subscriber, abort);\n    let requiredHead: string | Promise<string> | undefined;\n    let uncoveredMinWatermark: string | undefined;\n    let committed = false;\n    try {\n      await this.#cleanupGuard.runWhilePurgeBlocked(() => {\n        this.#throwIfAborted(abort.signal);\n        // The router's eligibility inspection happens before this async guard\n        // is acquired. Recheck the exact boundary here: an in-flight purge can\n        // have advanced the minimum while registration waited. Once added to\n        // the Forwarder, the subscriber's ACK prevents any later purge from\n        // crossing this boundary.\n        const plan = this.#reader.plan(subscriber.watermark);\n        if (plan.kind === 'too-old') {\n          uncoveredMinWatermark = plan.minWatermark;\n          return;\n        }\n        requiredHead = captureRequiredHead();\n        this.#forwarder.add(subscriber);\n        committed = true;\n      });\n    } catch (error) {\n      this.#catchups.delete(subscriber);\n      if (abort.signal.aborted) {\n        return {kind: 'handled'};\n      }\n      if (!committed) {\n        // Nothing was committed to SQLite -- the subscriber never reached\n        // Forwarder.add() -- so PG catchup is still available. Failing here\n        // would send the client a terminal ['error', ...], which\n        // IncrementalSyncer answers with a full litestream replica restore,\n        // for a fallback that was still open. Trip the breaker as well: this\n        // is a reader failure like any other, and without it the retry picks\n        // SQLite again and the reset repeats.\n        this.#onFailure?.('reader-error');\n        return {kind: 'declined', error};\n      }\n      this.#lc.error?.(\n        `error while registering SQLite catchup subscriber ${subscriber.id}`,\n        error,\n      );\n      subscriber.fail(error);\n      return {kind: 'handled'};\n    }\n    if (uncoveredMinWatermark !== undefined) {\n      this.#catchups.delete(subscriber);\n      return {kind: 'uncovered', minWatermark: uncoveredMinWatermark};\n    }\n\n    void this.#run(\n      subscriber,\n      requiredHead as string | Promise<string>,\n      abort,\n      request.logWarm,\n    );\n    return {kind: 'registered'};\n  }\n\n  /**\n   * Notes that the change-log writer has committed `watermark`, i.e. that the\n   * log's head has advanced to it. The writer runs in this process, so this is\n   * the event itself rather than a subscriber's ACK of it.\n   *\n   * It is a wakeup, not an authority: `plan()` still decides what can be read.\n   * A notification that never arrives costs the barrier a poll interval, and one\n   * that arrives early costs an extra `plan()` call.\n   */\n  onChangeLogCommit(watermark: string): void {\n    for (const waiter of this.#commitWaiters) {\n      if (watermark >= waiter.watermark) {\n        this.#commitWaiters.delete(waiter);\n        waiter.resolve();\n      }\n    }\n  }\n\n  remove(subscriber: Subscriber): void {\n    this.#catchups.get(subscriber)?.abort();\n    this.#catchups.delete(subscriber);\n    this.#forwarder.remove(subscriber);\n  }\n\n  close(): void {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    for (const [subscriber, abort] of this.#catchups) {\n      abort.abort();\n      this.#forwarder.remove(subscriber);\n    }\n    this.#catchups.clear();\n    this.#reader.close();\n  }\n\n  [Symbol.dispose](): void {\n    this.close();\n  }\n\n  async #run(\n    subscriber: Subscriber,\n    requiredHead: string | Promise<string>,\n    abort: AbortController,\n    logWarm: boolean | undefined,\n  ): Promise<void> {\n    const {signal} = abort;\n    const catchupStart = this.#now();\n    let outcome = 'aborted';\n    let backlogPeakBytes = 0;\n    try {\n      const requiredHeadStart = this.#now();\n      const required = await this.#awaitRequiredHead(requiredHead, signal);\n      const requiredHeadWaitMs = this.#now() - requiredHeadStart;\n      if (requiredHeadWaitMs > this.#barrierTimeoutMs) {\n        this.#lc.info?.(\n          `waited ${requiredHeadWaitMs} ms for the forwarded transaction to ` +\n            `commit before starting the SQLite barrier for ${subscriber.id}`,\n        );\n      }\n      // The barrier bounds replica lag, not upstream transaction duration, so\n      // its deadline starts once the required head is known. Sharing one\n      // deadline with the wait above would fail subscribers that registered\n      // during a transaction longer than barrierTimeoutMs, however current\n      // the replica is.\n      const deadline = this.#now() + this.#barrierTimeoutMs;\n      const barrierStart = this.#now();\n      let plan: CatchupPlan;\n      try {\n        plan = await this.#waitForPlan(subscriber, required, deadline, signal);\n      } finally {\n        // Record failed and aborted barriers too; their wait distribution is\n        // at least as operationally important as the successful path.\n        this.#barrierWaitDuration.recordMs(this.#now() - barrierStart);\n      }\n      this.#throwIfAborted(signal);\n\n      if (plan.kind === 'too-old') {\n        outcome = 'too-old';\n        const message =\n          `earliest supported watermark is ${plan.minWatermark} ` +\n          `(requested ${subscriber.watermark})`;\n        this.#lc.warn?.(\n          `rejecting subscriber at watermark ${subscriber.watermark} ` +\n            `(earliest watermark: ${plan.minWatermark})`,\n        );\n        subscriber.close(ErrorType.WatermarkTooOld, message);\n        return;\n      }\n\n      let count = 0;\n      let bytes = 0;\n      const start = this.#now();\n      if (plan.kind === 'range') {\n        let lastBatchConsumed: Promise<unknown> | undefined;\n        for await (const changes of this.#reader.read(\n          subscriber.watermark,\n          plan.headWatermark,\n          this.#batchSize,\n          signal,\n        )) {\n          const waitStart = this.#now();\n          await lastBatchConsumed;\n          const elapsed = this.#now() - waitStart;\n          if (lastBatchConsumed) {\n            this.#lc[elapsed > 100 ? 'info' : 'debug']?.(\n              `waited ${elapsed.toFixed(3)} ms for ${subscriber.id} to consume ` +\n                `the previous SQLite catchup batch`,\n            );\n          }\n          this.#throwIfAborted(signal);\n          for (const change of changes) {\n            lastBatchConsumed = subscriber.catchup(change);\n            count++;\n            bytes += Buffer.byteLength(change[2]);\n          }\n          backlogPeakBytes = Math.max(\n            backlogPeakBytes,\n            subscriber.getStats().backlogBytes,\n          );\n        }\n        await lastBatchConsumed;\n        outcome = 'range';\n      } else {\n        outcome = 'ahead';\n        this.#lc.warn?.(\n          `subscriber ${subscriber.id} at watermark ${subscriber.watermark} ` +\n            `is ahead of the SQLite change-log head ${plan.headWatermark}; ` +\n            `waiting for the replica to catch up`,\n        );\n      }\n\n      this.#throwIfAborted(signal);\n      this.#lc.info?.(\n        `caught up ${subscriber.id} from SQLite with ${count} changes ` +\n          `(${this.#now() - start} ms)`,\n      );\n      const attributes = {\n        classification: outcome,\n        log_warm: logWarm ?? 'unknown',\n      };\n      this.#catchupRows.add(count, attributes);\n      this.#catchupBytes.add(bytes, attributes);\n      // Keep buffering live sends until the asynchronous backlog drain has\n      // established its ordering boundary.\n      void subscriber.setCaughtUp();\n    } catch (error) {\n      if (signal.aborted) {\n        return;\n      }\n      if (error instanceof SQLiteChangeLogBarrierError) {\n        if (error instanceof SQLiteChangeLogBarrierTimeoutError) {\n          this.#barrierTimeouts.add(1);\n          outcome = 'barrier-timeout';\n          this.#onFailure?.('barrier-timeout');\n        } else {\n          // No #onFailure: a backlog high water mark is a property of this\n          // one subscriber's consumption, not of the log. Tripping the\n          // process-wide breaker for it would take every task in the shard\n          // off SQLite, which a client that reconnects and stalls again\n          // could hold open indefinitely.\n          outcome = 'barrier-backlog';\n        }\n        // Giving up on the barrier means the replica has not caught up *yet*,\n        // not that it cannot serve this subscriber. End the subscription\n        // cleanly so the client reconnects and retries: IncrementalSyncer\n        // treats any ['error', ...] message as terminal and restores a fresh\n        // replica from litestream, whereas a clean end backs off and\n        // re-subscribes. If retries keep failing until the change log is purged\n        // past the subscriber's watermark, plan() returns 'too-old', which is\n        // terminal by design.\n        this.#lc.warn?.(\n          `ending subscription for ${subscriber.id} to retry SQLite catchup`,\n          error,\n        );\n        subscriber.close();\n        return;\n      }\n      outcome = 'reader-error';\n      this.#onFailure?.('reader-error');\n      this.#lc.error?.(\n        `error while catching up subscriber ${subscriber.id} from SQLite`,\n        error,\n      );\n      subscriber.fail(error);\n    } finally {\n      backlogPeakBytes = Math.max(\n        backlogPeakBytes,\n        subscriber.getStats().backlogBytes,\n      );\n      const attributes = {\n        classification: outcome,\n        log_warm: logWarm ?? 'unknown',\n      };\n      this.#catchupResults.add(1, attributes);\n      this.#catchupDuration.recordMs(this.#now() - catchupStart, attributes);\n      this.#catchupBacklogPeak.record(backlogPeakBytes, attributes);\n      if (this.#catchups.get(subscriber) === abort) {\n        this.#catchups.delete(subscriber);\n      }\n    }\n  }\n\n  /**\n   * Waits for the transaction that was in flight at registration to finish.\n   *\n   * This wait is bounded by the upstream transaction rather than by\n   * `barrierTimeoutMs`: the completion settles on both commit and rollback,\n   * and an interrupted change stream forwards a synthetic rollback, so it\n   * always settles. Aborting -- the subscriber disconnecting or the service\n   * shutting down -- is what releases it early.\n   */\n  async #awaitRequiredHead(\n    requiredHead: string | Promise<string>,\n    signal: AbortSignal,\n  ): Promise<string> {\n    if (typeof requiredHead === 'string') {\n      return requiredHead;\n    }\n    this.#throwIfAborted(signal);\n    let onAbort: (() => void) | undefined;\n    try {\n      return await Promise.race([\n        requiredHead,\n        new Promise<never>((_, reject) => {\n          onAbort = () =>\n            reject(new AbortError('SQLite change-log catchup aborted'));\n          signal.addEventListener('abort', onAbort, {once: true});\n        }),\n      ]);\n    } finally {\n      if (onAbort) {\n        signal.removeEventListener('abort', onAbort);\n      }\n    }\n  }\n\n  async #waitForPlan(\n    subscriber: Subscriber,\n    requiredHead: string,\n    deadline: number,\n    signal: AbortSignal,\n  ): Promise<Exclude<CatchupPlan, {kind: 'not-ready'}>> {\n    // Race backlog growth once around the whole polling loop. Attaching the\n    // same unresolved promise to every per-poll race would retain one reaction\n    // per iteration until the subscriber's backlog eventually filled.\n    const backlogFull = subscriber.whenBacklogFull();\n    const barrier = new AbortController();\n    const abortBarrier = () => barrier.abort();\n    signal.addEventListener('abort', abortBarrier, {once: true});\n    if (signal.aborted) {\n      barrier.abort();\n    }\n\n    let observedHead = subscriber.watermark;\n    try {\n      const planReady = (async () => {\n        while (true) {\n          this.#throwIfAborted(barrier.signal);\n          const plan = this.#reader.plan(subscriber.watermark);\n          // A not-ready log has no head to compare, so it can never satisfy\n          // the required head: keep waiting for the writer to create and\n          // reconcile it, and let the barrier deadline end the subscription\n          // cleanly for a retry. Selection declines a not-ready log before a\n          // subscriber gets here, and neither purging nor reconciliation can\n          // empty a log that had content, so this is a belt-and-braces path.\n          if (plan.kind !== 'not-ready') {\n            observedHead = plan.headWatermark;\n            if (plan.headWatermark >= requiredHead) {\n              return plan;\n            }\n          }\n          const remaining = deadline - this.#now();\n          if (remaining <= 0) {\n            throw new SQLiteChangeLogBarrierTimeoutError(\n              `timed out waiting for SQLite head ${observedHead} to ` +\n                `reach required head ${requiredHead}`,\n            );\n          }\n          await this.#awaitChangeLogProgress(\n            requiredHead,\n            remaining,\n            barrier.signal,\n          );\n        }\n      })();\n\n      const backlogOverflow = backlogFull.promise.then(() => {\n        this.#throwIfAborted(signal);\n        // close() also resolves backlog waiters so none are stranded. The\n        // backlog is cleared first, which distinguishes close from overflow.\n        if (!subscriber.backlogFull) {\n          return new Promise<never>(() => {});\n        }\n        this.#barrierWakeups.add(1, {'wakeup.source': 'backlog'});\n        // Bytes, not time, are what the barrier actually costs: past the high\n        // water mark this subscriber's send() no longer resolves, so it holds\n        // up every flush and can stall the replica this barrier waits on.\n        this.#barrierBacklogOverflows.add(1);\n        throw new SQLiteChangeLogBarrierBacklogError(\n          `backlog for subscriber ${subscriber.id} reached its high water ` +\n            `mark while waiting for SQLite head ${observedHead} to ` +\n            `reach required head ${requiredHead}`,\n        );\n      });\n\n      return await Promise.race([planReady, backlogOverflow]);\n    } finally {\n      signal.removeEventListener('abort', abortBarrier);\n      barrier.abort();\n      backlogFull.cancel();\n    }\n  }\n\n  /**\n   * Waits for the change-log writer to commit `requiredHead`, or the poll\n   * interval as a backstop.\n   *\n   * No notification can be missed by registering after `plan()` was read: the\n   * notification follows the commit it reports, so if the writer had already\n   * committed `requiredHead`, `plan()` would have seen it.\n   */\n  async #awaitChangeLogProgress(\n    requiredHead: string,\n    remainingMs: number,\n    signal: AbortSignal,\n  ): Promise<void> {\n    const committed = resolver<void>();\n    const waiter = {watermark: requiredHead, resolve: committed.resolve};\n    this.#commitWaiters.add(waiter);\n    // Scopes the backstop timer to this wait so that a commit cancels it rather\n    // than leaving it to fire against a barrier that has already moved on.\n    const poll = new AbortController();\n    const abortPoll = () => poll.abort();\n    signal.addEventListener('abort', abortPoll, {once: true});\n    try {\n      const wokenBy = await Promise.race([\n        committed.promise.then(() => 'commit' as const),\n        this.#sleep(\n          Math.min(this.#barrierPollIntervalMs, remainingMs),\n          poll.signal,\n        ).then(() => 'poll' as const),\n      ]);\n      this.#barrierWakeups.add(1, {'wakeup.source': wokenBy});\n    } finally {\n      this.#commitWaiters.delete(waiter);\n      signal.removeEventListener('abort', abortPoll);\n      poll.abort();\n    }\n  }\n\n  #throwIfAborted(signal: AbortSignal): void {\n    if (this.#closed || signal.aborted) {\n      throw new AbortError('SQLite change-log catchup aborted');\n    }\n  }\n}\n\ntype CommitWaiter = {watermark: string; resolve: () => void};\n\n/**\n * A barrier that gave up on the replica reaching the required head. Both\n * reasons mean \"not yet\", not \"never\", so both end the subscription cleanly\n * for a retry rather than failing it.\n */\nexport class SQLiteChangeLogBarrierError extends Error {}\n\n/** The replica did not reach the required head before the deadline. */\nexport class SQLiteChangeLogBarrierTimeoutError extends SQLiteChangeLogBarrierError {\n  readonly name = 'SQLiteChangeLogBarrierTimeoutError';\n}\n\n/** Waiting any longer would have cost more than reconnecting. */\nexport class SQLiteChangeLogBarrierBacklogError extends SQLiteChangeLogBarrierError {\n  readonly name = 'SQLiteChangeLogBarrierBacklogError';\n}\n"],"mappings":";;;;;AAuCA,IAAM,mCAAmC;AAmCzC,IAAM,qBAAkD,EACtD,uBAAsB,aAAY,QAAQ,QAAQ,SAAS,CAAC,EAC9D;;;;;;;;;;AAWA,IAAa,yBAAb,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA,mBAA4B,mBAC1B,eACA,sCACA,0EACF;CACA,2BAAoC,mBAClC,eACA,+CACA,sIAEF;CACA,kBAA2B,mBACzB,eACA,qCACA,iLAGF;CACA,kBAA2B,mBACzB,eACA,qCACA,0CACF;CACA,eAAwB,mBACtB,eACA,kCACA,wDACF;CACA,gBAAyB,mBACvB,eACA,mCACA;EACE,aAAa;EACb,MAAM;CACR,CACF;CACA,mBAA4B,4BAC1B,eACA,sCACA,0CACF;CACA,uBAAgC,4BAC9B,eACA,2CACA,yDACF;CACA,sBAA+B,0BAC7B,eACA,gDACA;EACE,aACE;EACF,MAAM;EACN,kBAAkB;GAChB;GAAG;GAAM;GAAQ;GAAQ;GAAS;GAAW;GAAW;EAC1D;CACF,CACF;CACA,4BAAqB,IAAI,IAAiC;CAC1D,iCAA0B,IAAI,IAAkB;CAChD,UAAU;CAEV,YACE,IACA,WACA,QACA,MACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAKC,aAAa;EAClB,KAAKC,UAAU;EACf,KAAKC,aAAa,KAAK;EACvB,KAAKC,oBAAoB,KAAK;EAC9B,KAAKC,yBACH,KAAK,yBAAyB;EAChC,KAAKC,gBAAgB,KAAK,gBAAgB;EAC1C,KAAKC,SAAS,KAAK,SAAS;EAC5B,KAAKC,OAAO,KAAK,OAAO,KAAK;EAC7B,KAAKC,aAAa,KAAK;CACzB;;;;;;CAOA,MAAM,QACJ,YACA,qBACA,UAAyC,CAAC,GACG;EAC7C,IAAI,KAAKY,SAAS;GAChB,WAAW,KAAK,IAAI,WAAW,qCAAqC,CAAC;GACrE,OAAO,EAAC,MAAM,UAAS;EACzB;EAEA,MAAM,QAAQ,IAAI,gBAAgB;EAClC,KAAKF,UAAU,IAAI,YAAY,KAAK;EACpC,IAAI;EACJ,IAAI;EACJ,IAAI,YAAY;EAChB,IAAI;GACF,MAAM,KAAKb,cAAc,2BAA2B;IAClD,KAAKgB,gBAAgB,MAAM,MAAM;IAMjC,MAAM,OAAO,KAAKpB,QAAQ,KAAK,WAAW,SAAS;IACnD,IAAI,KAAK,SAAS,WAAW;KAC3B,wBAAwB,KAAK;KAC7B;IACF;IACA,eAAe,oBAAoB;IACnC,KAAKD,WAAW,IAAI,UAAU;IAC9B,YAAY;GACd,CAAC;EACH,SAAS,OAAO;GACd,KAAKkB,UAAU,OAAO,UAAU;GAChC,IAAI,MAAM,OAAO,SACf,OAAO,EAAC,MAAM,UAAS;GAEzB,IAAI,CAAC,WAAW;IAQd,KAAKV,aAAa,cAAc;IAChC,OAAO;KAAC,MAAM;KAAY;IAAK;GACjC;GACA,KAAKT,IAAI,QACP,qDAAqD,WAAW,MAChE,KACF;GACA,WAAW,KAAK,KAAK;GACrB,OAAO,EAAC,MAAM,UAAS;EACzB;EACA,IAAI,0BAA0B,KAAA,GAAW;GACvC,KAAKmB,UAAU,OAAO,UAAU;GAChC,OAAO;IAAC,MAAM;IAAa,cAAc;GAAqB;EAChE;EAEA,KAAUI,KACR,YACA,cACA,OACA,QAAQ,OACV;EACA,OAAO,EAAC,MAAM,aAAY;CAC5B;;;;;;;;;;CAWA,kBAAkB,WAAyB;EACzC,KAAK,MAAM,UAAU,KAAKH,gBACxB,IAAI,aAAa,OAAO,WAAW;GACjC,KAAKA,eAAe,OAAO,MAAM;GACjC,OAAO,QAAQ;EACjB;CAEJ;CAEA,OAAO,YAA8B;EACnC,KAAKD,UAAU,IAAI,UAAU,GAAG,MAAM;EACtC,KAAKA,UAAU,OAAO,UAAU;EAChC,KAAKlB,WAAW,OAAO,UAAU;CACnC;CAEA,QAAc;EACZ,IAAI,KAAKoB,SACP;EAEF,KAAKA,UAAU;EACf,KAAK,MAAM,CAAC,YAAY,UAAU,KAAKF,WAAW;GAChD,MAAM,MAAM;GACZ,KAAKlB,WAAW,OAAO,UAAU;EACnC;EACA,KAAKkB,UAAU,MAAM;EACrB,KAAKjB,QAAQ,MAAM;CACrB;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,MAAM;CACb;CAEA,MAAMqB,KACJ,YACA,cACA,OACA,SACe;EACf,MAAM,EAAC,WAAU;EACjB,MAAM,eAAe,KAAKf,KAAK;EAC/B,IAAI,UAAU;EACd,IAAI,mBAAmB;EACvB,IAAI;GACF,MAAM,oBAAoB,KAAKA,KAAK;GACpC,MAAM,WAAW,MAAM,KAAKgB,mBAAmB,cAAc,MAAM;GACnE,MAAM,qBAAqB,KAAKhB,KAAK,IAAI;GACzC,IAAI,qBAAqB,KAAKJ,mBAC5B,KAAKJ,IAAI,OACP,UAAU,mBAAmB,qFACsB,WAAW,IAChE;GAOF,MAAM,WAAW,KAAKQ,KAAK,IAAI,KAAKJ;GACpC,MAAM,eAAe,KAAKI,KAAK;GAC/B,IAAI;GACJ,IAAI;IACF,OAAO,MAAM,KAAKiB,aAAa,YAAY,UAAU,UAAU,MAAM;GACvE,UAAU;IAGR,KAAKR,qBAAqB,SAAS,KAAKT,KAAK,IAAI,YAAY;GAC/D;GACA,KAAKc,gBAAgB,MAAM;GAE3B,IAAI,KAAK,SAAS,WAAW;IAC3B,UAAU;IACV,MAAM,UACJ,mCAAmC,KAAK,aAAa,cACvC,WAAW,UAAU;IACrC,KAAKtB,IAAI,OACP,qCAAqC,WAAW,UAAU,wBAChC,KAAK,aAAa,EAC9C;IACA,WAAW,MAAM,GAA2B,OAAO;IACnD;GACF;GAEA,IAAI,QAAQ;GACZ,IAAI,QAAQ;GACZ,MAAM,QAAQ,KAAKQ,KAAK;GACxB,IAAI,KAAK,SAAS,SAAS;IACzB,IAAI;IACJ,WAAW,MAAM,WAAW,KAAKN,QAAQ,KACvC,WAAW,WACX,KAAK,eACL,KAAKC,YACL,MACF,GAAG;KACD,MAAM,YAAY,KAAKK,KAAK;KAC5B,MAAM;KACN,MAAM,UAAU,KAAKA,KAAK,IAAI;KAC9B,IAAI,mBACF,KAAKR,IAAI,UAAU,MAAM,SAAS,WAChC,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,WAAW,GAAG,8CAEvD;KAEF,KAAKsB,gBAAgB,MAAM;KAC3B,KAAK,MAAM,UAAU,SAAS;MAC5B,oBAAoB,WAAW,QAAQ,MAAM;MAC7C;MACA,SAAS,OAAO,WAAW,OAAO,EAAE;KACtC;KACA,mBAAmB,KAAK,IACtB,kBACA,WAAW,SAAS,EAAE,YACxB;IACF;IACA,MAAM;IACN,UAAU;GACZ,OAAO;IACL,UAAU;IACV,KAAKtB,IAAI,OACP,cAAc,WAAW,GAAG,gBAAgB,WAAW,UAAU,0CACrB,KAAK,cAAc,sCAEjE;GACF;GAEA,KAAKsB,gBAAgB,MAAM;GAC3B,KAAKtB,IAAI,OACP,aAAa,WAAW,GAAG,oBAAoB,MAAM,YAC/C,KAAKQ,KAAK,IAAI,MAAM,KAC5B;GACA,MAAM,aAAa;IACjB,gBAAgB;IAChB,UAAU,WAAW;GACvB;GACA,KAAKM,aAAa,IAAI,OAAO,UAAU;GACvC,KAAKC,cAAc,IAAI,OAAO,UAAU;GAGxC,WAAgB,YAAY;EAC9B,SAAS,OAAO;GACd,IAAI,OAAO,SACT;GAEF,IAAI,iBAAiB,6BAA6B;IAChD,IAAI,iBAAiB,oCAAoC;KACvD,KAAKL,iBAAiB,IAAI,CAAC;KAC3B,UAAU;KACV,KAAKD,aAAa,iBAAiB;IACrC,OAME,UAAU;IAUZ,KAAKT,IAAI,OACP,2BAA2B,WAAW,GAAG,2BACzC,KACF;IACA,WAAW,MAAM;IACjB;GACF;GACA,UAAU;GACV,KAAKS,aAAa,cAAc;GAChC,KAAKT,IAAI,QACP,sCAAsC,WAAW,GAAG,eACpD,KACF;GACA,WAAW,KAAK,KAAK;EACvB,UAAU;GACR,mBAAmB,KAAK,IACtB,kBACA,WAAW,SAAS,EAAE,YACxB;GACA,MAAM,aAAa;IACjB,gBAAgB;IAChB,UAAU,WAAW;GACvB;GACA,KAAKa,gBAAgB,IAAI,GAAG,UAAU;GACtC,KAAKG,iBAAiB,SAAS,KAAKR,KAAK,IAAI,cAAc,UAAU;GACrE,KAAKU,oBAAoB,OAAO,kBAAkB,UAAU;GAC5D,IAAI,KAAKC,UAAU,IAAI,UAAU,MAAM,OACrC,KAAKA,UAAU,OAAO,UAAU;EAEpC;CACF;;;;;;;;;;CAWA,MAAMK,mBACJ,cACA,QACiB;EACjB,IAAI,OAAO,iBAAiB,UAC1B,OAAO;EAET,KAAKF,gBAAgB,MAAM;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,QAAQ,KAAK,CACxB,cACA,IAAI,SAAgB,GAAG,WAAW;IAChC,gBACE,OAAO,IAAI,WAAW,mCAAmC,CAAC;IAC5D,OAAO,iBAAiB,SAAS,SAAS,EAAC,MAAM,KAAI,CAAC;GACxD,CAAC,CACH,CAAC;EACH,UAAU;GACR,IAAI,SACF,OAAO,oBAAoB,SAAS,OAAO;EAE/C;CACF;CAEA,MAAMG,aACJ,YACA,cACA,UACA,QACoD;EAIpD,MAAM,cAAc,WAAW,gBAAgB;EAC/C,MAAM,UAAU,IAAI,gBAAgB;EACpC,MAAM,qBAAqB,QAAQ,MAAM;EACzC,OAAO,iBAAiB,SAAS,cAAc,EAAC,MAAM,KAAI,CAAC;EAC3D,IAAI,OAAO,SACT,QAAQ,MAAM;EAGhB,IAAI,eAAe,WAAW;EAC9B,IAAI;GACF,MAAM,aAAa,YAAY;IAC7B,OAAO,MAAM;KACX,KAAKH,gBAAgB,QAAQ,MAAM;KACnC,MAAM,OAAO,KAAKpB,QAAQ,KAAK,WAAW,SAAS;KAOnD,IAAI,KAAK,SAAS,aAAa;MAC7B,eAAe,KAAK;MACpB,IAAI,KAAK,iBAAiB,cACxB,OAAO;KAEX;KACA,MAAM,YAAY,WAAW,KAAKM,KAAK;KACvC,IAAI,aAAa,GACf,MAAM,IAAI,mCACR,qCAAqC,aAAa,0BACzB,cAC3B;KAEF,MAAM,KAAKkB,wBACT,cACA,WACA,QAAQ,MACV;IACF;GACF,GAAG;GAEH,MAAM,kBAAkB,YAAY,QAAQ,WAAW;IACrD,KAAKJ,gBAAgB,MAAM;IAG3B,IAAI,CAAC,WAAW,aACd,OAAO,IAAI,cAAqB,CAAC,CAAC;IAEpC,KAAKV,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,UAAS,CAAC;IAIxD,KAAKD,yBAAyB,IAAI,CAAC;IACnC,MAAM,IAAI,mCACR,0BAA0B,WAAW,GAAG,6DACA,aAAa,0BAC5B,cAC3B;GACF,CAAC;GAED,OAAO,MAAM,QAAQ,KAAK,CAAC,WAAW,eAAe,CAAC;EACxD,UAAU;GACR,OAAO,oBAAoB,SAAS,YAAY;GAChD,QAAQ,MAAM;GACd,YAAY,OAAO;EACrB;CACF;;;;;;;;;CAUA,MAAMe,wBACJ,cACA,aACA,QACe;EACf,MAAM,YAAY,SAAe;EACjC,MAAM,SAAS;GAAC,WAAW;GAAc,SAAS,UAAU;EAAO;EACnE,KAAKN,eAAe,IAAI,MAAM;EAG9B,MAAM,OAAO,IAAI,gBAAgB;EACjC,MAAM,kBAAkB,KAAK,MAAM;EACnC,OAAO,iBAAiB,SAAS,WAAW,EAAC,MAAM,KAAI,CAAC;EACxD,IAAI;GACF,MAAM,UAAU,MAAM,QAAQ,KAAK,CACjC,UAAU,QAAQ,WAAW,QAAiB,GAC9C,KAAKb,OACH,KAAK,IAAI,KAAKF,wBAAwB,WAAW,GACjD,KAAK,MACP,EAAE,WAAW,MAAe,CAC9B,CAAC;GACD,KAAKO,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,QAAO,CAAC;EACxD,UAAU;GACR,KAAKQ,eAAe,OAAO,MAAM;GACjC,OAAO,oBAAoB,SAAS,SAAS;GAC7C,KAAK,MAAM;EACb;CACF;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAKC,WAAW,OAAO,SACzB,MAAM,IAAI,WAAW,mCAAmC;CAE5D;AACF;;;;;;AASA,IAAa,8BAAb,cAAiD,MAAM,CAAC;;AAGxD,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB;;AAGA,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.d.ts

@@ -0,0 +1,83 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { ShardID } from '../../types/shards.ts';
+import { type ChangeLogIdentity } from '../replicator/change-log-db.ts';
+import { type ChangeLogEntry } from './change-log-codec.ts';
+/**
+ * Result for one committed transaction.
+ *
+ * `mismatch` covers missing, extra, changed, or reordered rows.
+ * `inconclusive` means that store bounds changed during the comparison.
+ */
+export type CompareOutcome = 'match' | 'mismatch' | 'inconclusive' | 'deferred' | 'oversized';
+export type CompareSkipReason = 'stopped' | 'log-rebuilt' | 'log-unavailable' | 'ineligible-schema' | 'ineligible-identity' | 'cold-log' | 'nothing-to-compare' | 'error';
+export type CompareCycleResult = {
+    readonly kind: 'skipped';
+    readonly reason: CompareSkipReason;
+} | {
+    readonly kind: 'compared';
+    /** Exclusive lower bound of the compared range. */
+    readonly fromWatermark: string;
+    /** Inclusive upper bound handled. The next cycle resumes from here. */
+    readonly throughWatermark: string;
+    /** Committed transactions handled through `throughWatermark`. */
+    readonly transactions: number;
+    /** Transactions with compared catchup output. */
+    readonly sampled: number;
+    readonly matched: number;
+    readonly mismatched: number;
+    /** Suspects rejected because store bounds changed. */
+    readonly inconclusive: number;
+    /** Sampled transactions deferred to a fresh budget. */
+    readonly deferred: number;
+    /** Sampled transactions that exceed a fresh budget. */
+    readonly oversized: number;
+};
+/** Provides the Postgres side of the catchup comparison. */
+export interface PGChangeLogRangeReader {
+    getCatchupBounds(): Promise<{
+        minWatermark: string | null;
+        lastWatermark: string;
+    }>;
+    listCommitWatermarks(afterWatermark: string, throughWatermark: string, limit: number): Promise<string[]>;
+    readCatchupRange(afterWatermark: string, throughWatermark: string, batchRows?: number | undefined): AsyncIterable<ChangeLogEntry[]>;
+}
+export type SQLiteChangeLogCompareOptions = {
+    /** Stable percentage of transactions selected for payload comparison. */
+    readonly comparePercent: number;
+    /** Warm-up time after a seed. The comparator skips a younger log. */
+    readonly retentionMs: number;
+    readonly intervalMs?: number | undefined;
+    readonly maxTransactionsPerCycle?: number | undefined;
+    /** Maximum catchup rows read from each store in one cycle. */
+    readonly maxRowsPerSourcePerCycle?: number | undefined;
+    /** Maximum catchup bytes read from each store in one cycle. */
+    readonly maxBytesPerSourcePerCycle?: number | undefined;
+    readonly readBatchRows?: number | undefined;
+    /** Clock for the warm-up period. Defaults to `Date.now`. */
+    readonly now?: (() => number) | undefined;
+    readonly setTimeoutFn?: typeof setTimeout | undefined;
+    readonly clearTimeoutFn?: typeof clearTimeout | undefined;
+    /** Yields between sampled transactions. Defaults to `setImmediate`. */
+    readonly yieldFn?: (() => Promise<void>) | undefined;
+};
+/**
+ * Compares SQLite and Postgres catchup output without affecting subscribers.
+ *
+ * Each cycle compares complete ranges through the lower store head.
+ * It checks every transaction for presence and samples payloads.
+ * A bounds change makes the affected result `inconclusive`.
+ * Postgres remains authoritative.
+ */
+export declare class SQLiteChangeLogComparator {
+    #private;
+    constructor(lc: LogContext, shard: ShardID, changeLogFile: string, identity: ChangeLogIdentity, pg: PGChangeLogRangeReader, opts: SQLiteChangeLogCompareOptions);
+    /**
+     * Runs one cycle. Concurrent calls share the same promise.
+     * Errors produce `skipped: 'error'`.
+     */
+    compareOnce(): Promise<CompareCycleResult>;
+    stop(): void;
+    /** Invalidates cycles that can still read a replaced file. */
+    invalidate(): void;
+}
+//# sourceMappingURL=sqlite-change-log-comparator.d.ts.map

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite-change-log-comparator.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-comparator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAIL,KAAK,iBAAiB,EAEvB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AA8B/B;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GACtB,OAAO,GACP,UAAU,GACV,cAAc,GACd,UAAU,GACV,WAAW,CAAC;AAEhB,MAAM,MAAM,iBAAiB,GAEzB,SAAS,GAET,aAAa,GAEb,iBAAiB,GAEjB,mBAAmB,GAEnB,qBAAqB,GAErB,UAAU,GAEV,oBAAoB,GAEpB,OAAO,CAAC;AAEZ,MAAM,MAAM,kBAAkB,GAC1B;IAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAA;CAAC,GAC9D;IACE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,mDAAmD;IACnD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,uEAAuE;IACvE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,iEAAiE;IACjE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,iDAAiD;IACjD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,sDAAsD;IACtD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,uDAAuD;IACvD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,uDAAuD;IACvD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEN,4DAA4D;AAC5D,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,IAAI,OAAO,CAAC;QAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,oBAAoB,CAClB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACrB,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,GAC7B,aAAa,CAAC,cAAc,EAAE,CAAC,CAAC;CACpC;AAED,MAAM,MAAM,6BAA6B,GAAG;IAC1C,yEAAyE;IACzE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,qEAAqE;IACrE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtD,8DAA8D;IAC9D,QAAQ,CAAC,wBAAwB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvD,+DAA+D;IAC/D,QAAQ,CAAC,yBAAyB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxD,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5C,4DAA4D;IAC5D,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IACtD,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;IAC1D,uEAAuE;IACvE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC;CACtD,CAAC;AAyCF;;;;;;;GAOG;AACH,qBAAa,yBAAyB;;gBAuClC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,iBAAiB,EAC3B,EAAE,EAAE,sBAAsB,EAC1B,IAAI,EAAE,6BAA6B;IAiBrC;;;OAGG;IACH,WAAW,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAmB1C,IAAI,IAAI,IAAI;IAQZ,8DAA8D;IAC9D,UAAU,IAAI,IAAI;CAifnB"}