@rocicorp/zero 1.10.0-canary.2 → 1.10.0-canary.20

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-catchup.js","names":["#lc","#forwarder","#reader","#batchSize","#barrierTimeoutMs","#barrierPollIntervalMs","#cleanupGuard","#sleep","#now","#barrierTimeouts","#barrierBacklogOverflows","#barrierWakeups","#catchups","#commitWaiters","#closed","#throwIfAborted","#run","#awaitRequiredHead","#waitForPlan","#awaitChangeLogProgress"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {Forwarder} from './forwarder.ts';\nimport type {CatchupPlan} from './sqlite-change-log-reader.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport interface SQLiteChangeLogCatchupReader {\n  plan(fromWatermark: string): CatchupPlan;\n  read(\n    fromWatermark: string,\n    throughWatermark: string,\n    batchSize: number,\n    signal?: AbortSignal,\n  ): AsyncIterable<readonly WatermarkedChange[]>;\n  close(): void;\n}\n\n/**\n * Slice 9 replaces this no-op implementation with a guard that waits for an\n * in-flight purge and blocks new purge dispatch while the subscriber's ACK is\n * made visible to the Forwarder.\n */\nexport interface SQLiteChangeLogCleanupGuard {\n  runWhilePurgeBlocked<T>(register: () => T): Promise<T>;\n}\n\n// How long the barrier waits for the change-log writer's commit notification\n// before re-reading the log itself. The notification normally arrives first, so\n// this is a backstop for the windows in which none is coming: the writer is\n// disabled, or it is between stream connections.\nconst DEFAULT_BARRIER_POLL_INTERVAL_MS = 1000;\n\nexport type SQLiteChangeLogCatchupOptions = {\n  batchSize: number;\n  barrierTimeoutMs: number;\n  barrierPollIntervalMs?: number | undefined;\n  cleanupGuard?: SQLiteChangeLogCleanupGuard | undefined;\n  sleep?: typeof sleep | undefined;\n  now?: (() => number) | undefined;\n};\n\nconst NOOP_CLEANUP_GUARD: SQLiteChangeLogCleanupGuard = {\n  runWhilePurgeBlocked: register => Promise.resolve(register()),\n};\n\n/**\n * Coordinates a serving subscriber's gap-free transition from replica-local\n * SQLite catchup to the Forwarder's live stream.\n *\n * Registration and required-head capture happen in one synchronous callback.\n * The subscriber therefore either sees a transaction in SQLite catchup or in\n * its live backlog (duplicates across that boundary are filtered by\n * Subscriber), never in neither place.\n */\nexport class SQLiteChangeLogCatchup implements Disposable {\n  readonly #lc: LogContext;\n  readonly #forwarder: Forwarder;\n  readonly #reader: SQLiteChangeLogCatchupReader;\n  readonly #batchSize: number;\n  readonly #barrierTimeoutMs: number;\n  readonly #barrierPollIntervalMs: number;\n  readonly #cleanupGuard: SQLiteChangeLogCleanupGuard;\n  readonly #sleep: typeof sleep;\n  readonly #now: () => number;\n  readonly #barrierTimeouts = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_timeouts',\n    'SQLite change-log catchups that timed out waiting for the required head.',\n  );\n  readonly #barrierBacklogOverflows = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_backlog_overflows',\n    'SQLite change-log catchups abandoned because the subscriber backlog ' +\n      'reached its high water mark while waiting for the required head.',\n  );\n  readonly #barrierWakeups = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_wakeups',\n    'SQLite catchup barrier waits, labeled by what ended the wait. A ' +\n      'population dominated by \"poll\" means the change-log writer\\'s commit ' +\n      'notification is not reaching the barrier.',\n  );\n  readonly #catchups = new Map<Subscriber, AbortController>();\n  readonly #commitWaiters = new Set<CommitWaiter>();\n  #closed = false;\n\n  constructor(\n    lc: LogContext,\n    forwarder: Forwarder,\n    reader: SQLiteChangeLogCatchupReader,\n    opts: SQLiteChangeLogCatchupOptions,\n  ) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-catchup');\n    this.#forwarder = forwarder;\n    this.#reader = reader;\n    this.#batchSize = opts.batchSize;\n    this.#barrierTimeoutMs = opts.barrierTimeoutMs;\n    this.#barrierPollIntervalMs =\n      opts.barrierPollIntervalMs ?? DEFAULT_BARRIER_POLL_INTERVAL_MS;\n    this.#cleanupGuard = opts.cleanupGuard ?? NOOP_CLEANUP_GUARD;\n    this.#sleep = opts.sleep ?? sleep;\n    this.#now = opts.now ?? Date.now;\n  }\n\n  /**\n   * Registers the subscriber before resolving. Catchup itself continues in\n   * the background so subscribe() does not wait for SQLite to reach the\n   * required head.\n   */\n  async catchup(\n    subscriber: Subscriber,\n    captureRequiredHead: () => string | Promise<string>,\n  ): Promise<void> {\n    if (this.#closed) {\n      subscriber.fail(new AbortError('SQLite change-log catchup is closed'));\n      return;\n    }\n\n    const abort = new AbortController();\n    this.#catchups.set(subscriber, abort);\n    let requiredHead: string | Promise<string> | undefined;\n    try {\n      await this.#cleanupGuard.runWhilePurgeBlocked(() => {\n        this.#throwIfAborted(abort.signal);\n        requiredHead = captureRequiredHead();\n        this.#forwarder.add(subscriber);\n      });\n    } catch (error) {\n      this.#catchups.delete(subscriber);\n      if (!abort.signal.aborted) {\n        this.#lc.error?.(\n          `error while registering SQLite catchup subscriber ${subscriber.id}`,\n          error,\n        );\n        subscriber.fail(error);\n      }\n      return;\n    }\n\n    void this.#run(subscriber, requiredHead as string | Promise<string>, abort);\n  }\n\n  /**\n   * Notes that the change-log writer has committed `watermark`, i.e. that the\n   * log's head has advanced to it. The writer runs in this process, so this is\n   * the event itself rather than a subscriber's ACK of it.\n   *\n   * It is a wakeup, not an authority: `plan()` still decides what can be read.\n   * A notification that never arrives costs the barrier a poll interval, and one\n   * that arrives early costs an extra `plan()` call.\n   */\n  onChangeLogCommit(watermark: string): void {\n    for (const waiter of this.#commitWaiters) {\n      if (watermark >= waiter.watermark) {\n        this.#commitWaiters.delete(waiter);\n        waiter.resolve();\n      }\n    }\n  }\n\n  remove(subscriber: Subscriber): void {\n    this.#catchups.get(subscriber)?.abort();\n    this.#catchups.delete(subscriber);\n    this.#forwarder.remove(subscriber);\n  }\n\n  close(): void {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    for (const [subscriber, abort] of this.#catchups) {\n      abort.abort();\n      this.#forwarder.remove(subscriber);\n    }\n    this.#catchups.clear();\n    this.#reader.close();\n  }\n\n  [Symbol.dispose](): void {\n    this.close();\n  }\n\n  async #run(\n    subscriber: Subscriber,\n    requiredHead: string | Promise<string>,\n    abort: AbortController,\n  ): Promise<void> {\n    const {signal} = abort;\n    try {\n      const requiredHeadStart = this.#now();\n      const required = await this.#awaitRequiredHead(requiredHead, signal);\n      const requiredHeadWaitMs = this.#now() - requiredHeadStart;\n      if (requiredHeadWaitMs > this.#barrierTimeoutMs) {\n        this.#lc.info?.(\n          `waited ${requiredHeadWaitMs} ms for the forwarded transaction to ` +\n            `commit before starting the SQLite barrier for ${subscriber.id}`,\n        );\n      }\n      // The barrier bounds replica lag, not upstream transaction duration, so\n      // its deadline starts once the required head is known. Sharing one\n      // deadline with the wait above would fail subscribers that registered\n      // during a transaction longer than barrierTimeoutMs, however current\n      // the replica is.\n      const deadline = this.#now() + this.#barrierTimeoutMs;\n      const plan = await this.#waitForPlan(\n        subscriber,\n        required,\n        deadline,\n        signal,\n      );\n      this.#throwIfAborted(signal);\n\n      if (plan.kind === 'too-old') {\n        const message =\n          `earliest supported watermark is ${plan.minWatermark} ` +\n          `(requested ${subscriber.watermark})`;\n        this.#lc.warn?.(\n          `rejecting subscriber at watermark ${subscriber.watermark} ` +\n            `(earliest watermark: ${plan.minWatermark})`,\n        );\n        subscriber.close(ErrorType.WatermarkTooOld, message);\n        return;\n      }\n\n      let count = 0;\n      const start = this.#now();\n      if (plan.kind === 'range') {\n        let lastBatchConsumed: Promise<unknown> | undefined;\n        for await (const changes of this.#reader.read(\n          subscriber.watermark,\n          plan.headWatermark,\n          this.#batchSize,\n          signal,\n        )) {\n          const waitStart = this.#now();\n          await lastBatchConsumed;\n          const elapsed = this.#now() - waitStart;\n          if (lastBatchConsumed) {\n            this.#lc[elapsed > 100 ? 'info' : 'debug']?.(\n              `waited ${elapsed.toFixed(3)} ms for ${subscriber.id} to consume ` +\n                `the previous SQLite catchup batch`,\n            );\n          }\n          this.#throwIfAborted(signal);\n          for (const change of changes) {\n            lastBatchConsumed = subscriber.catchup(change);\n            count++;\n          }\n        }\n        await lastBatchConsumed;\n      } else {\n        this.#lc.warn?.(\n          `subscriber ${subscriber.id} at watermark ${subscriber.watermark} ` +\n            `is ahead of the SQLite change-log head ${plan.headWatermark}; ` +\n            `waiting for the replica to catch up`,\n        );\n      }\n\n      this.#throwIfAborted(signal);\n      this.#lc.info?.(\n        `caught up ${subscriber.id} from SQLite with ${count} changes ` +\n          `(${this.#now() - start} ms)`,\n      );\n      // Keep buffering live sends until the asynchronous backlog drain has\n      // established its ordering boundary.\n      void subscriber.setCaughtUp();\n    } catch (error) {\n      if (signal.aborted) {\n        return;\n      }\n      if (error instanceof SQLiteChangeLogBarrierError) {\n        if (error instanceof SQLiteChangeLogBarrierTimeoutError) {\n          this.#barrierTimeouts.add(1);\n        }\n        // Giving up on the barrier means the replica has not caught up *yet*,\n        // not that it cannot serve this subscriber. End the subscription\n        // cleanly so the client reconnects and retries: IncrementalSyncer\n        // treats any ['error', ...] message as terminal and restores a fresh\n        // replica from litestream, whereas a clean end backs off and\n        // re-subscribes. If retries keep failing until the change log is purged\n        // past the subscriber's watermark, plan() returns 'too-old', which is\n        // terminal by design.\n        this.#lc.warn?.(\n          `ending subscription for ${subscriber.id} to retry SQLite catchup`,\n          error,\n        );\n        subscriber.close();\n        return;\n      }\n      this.#lc.error?.(\n        `error while catching up subscriber ${subscriber.id} from SQLite`,\n        error,\n      );\n      subscriber.fail(error);\n    } finally {\n      if (this.#catchups.get(subscriber) === abort) {\n        this.#catchups.delete(subscriber);\n      }\n    }\n  }\n\n  /**\n   * Waits for the transaction that was in flight at registration to finish.\n   *\n   * This wait is bounded by the upstream transaction rather than by\n   * `barrierTimeoutMs`: the completion settles on both commit and rollback,\n   * and an interrupted change stream forwards a synthetic rollback, so it\n   * always settles. Aborting -- the subscriber disconnecting or the service\n   * shutting down -- is what releases it early.\n   */\n  async #awaitRequiredHead(\n    requiredHead: string | Promise<string>,\n    signal: AbortSignal,\n  ): Promise<string> {\n    if (typeof requiredHead === 'string') {\n      return requiredHead;\n    }\n    this.#throwIfAborted(signal);\n    let onAbort: (() => void) | undefined;\n    try {\n      return await Promise.race([\n        requiredHead,\n        new Promise<never>((_, reject) => {\n          onAbort = () =>\n            reject(new AbortError('SQLite change-log catchup aborted'));\n          signal.addEventListener('abort', onAbort, {once: true});\n        }),\n      ]);\n    } finally {\n      if (onAbort) {\n        signal.removeEventListener('abort', onAbort);\n      }\n    }\n  }\n\n  async #waitForPlan(\n    subscriber: Subscriber,\n    requiredHead: string,\n    deadline: number,\n    signal: AbortSignal,\n  ): Promise<Exclude<CatchupPlan, {kind: 'not-ready'}>> {\n    // Race backlog growth once around the whole polling loop. Attaching the\n    // same unresolved promise to every per-poll race would retain one reaction\n    // per iteration until the subscriber's backlog eventually filled.\n    const backlogFull = subscriber.whenBacklogFull();\n    const barrier = new AbortController();\n    const abortBarrier = () => barrier.abort();\n    signal.addEventListener('abort', abortBarrier, {once: true});\n    if (signal.aborted) {\n      barrier.abort();\n    }\n\n    let observedHead = subscriber.watermark;\n    try {\n      const planReady = (async () => {\n        while (true) {\n          this.#throwIfAborted(barrier.signal);\n          const plan = this.#reader.plan(subscriber.watermark);\n          // A not-ready log has no head to compare, so it can never satisfy\n          // the required head: keep waiting for the writer to create and\n          // reconcile it, and let the barrier deadline end the subscription\n          // cleanly for a retry. Selection declines a not-ready log before a\n          // subscriber gets here, and neither purging nor reconciliation can\n          // empty a log that had content, so this is a belt-and-braces path.\n          if (plan.kind !== 'not-ready') {\n            observedHead = plan.headWatermark;\n            if (plan.headWatermark >= requiredHead) {\n              return plan;\n            }\n          }\n          const remaining = deadline - this.#now();\n          if (remaining <= 0) {\n            throw new SQLiteChangeLogBarrierTimeoutError(\n              `timed out waiting for SQLite head ${observedHead} to ` +\n                `reach required head ${requiredHead}`,\n            );\n          }\n          await this.#awaitChangeLogProgress(\n            requiredHead,\n            remaining,\n            barrier.signal,\n          );\n        }\n      })();\n\n      const backlogOverflow = backlogFull.promise.then(() => {\n        this.#throwIfAborted(signal);\n        // close() also resolves backlog waiters so none are stranded. The\n        // backlog is cleared first, which distinguishes close from overflow.\n        if (!subscriber.backlogFull) {\n          return new Promise<never>(() => {});\n        }\n        this.#barrierWakeups.add(1, {'wakeup.source': 'backlog'});\n        // Bytes, not time, are what the barrier actually costs: past the high\n        // water mark this subscriber's send() no longer resolves, so it holds\n        // up every flush and can stall the replica this barrier waits on.\n        this.#barrierBacklogOverflows.add(1);\n        throw new SQLiteChangeLogBarrierBacklogError(\n          `backlog for subscriber ${subscriber.id} reached its high water ` +\n            `mark while waiting for SQLite head ${observedHead} to ` +\n            `reach required head ${requiredHead}`,\n        );\n      });\n\n      return await Promise.race([planReady, backlogOverflow]);\n    } finally {\n      signal.removeEventListener('abort', abortBarrier);\n      barrier.abort();\n      backlogFull.cancel();\n    }\n  }\n\n  /**\n   * Waits for the change-log writer to commit `requiredHead`, or the poll\n   * interval as a backstop.\n   *\n   * No notification can be missed by registering after `plan()` was read: the\n   * notification follows the commit it reports, so if the writer had already\n   * committed `requiredHead`, `plan()` would have seen it.\n   */\n  async #awaitChangeLogProgress(\n    requiredHead: string,\n    remainingMs: number,\n    signal: AbortSignal,\n  ): Promise<void> {\n    const committed = resolver<void>();\n    const waiter = {watermark: requiredHead, resolve: committed.resolve};\n    this.#commitWaiters.add(waiter);\n    // Scopes the backstop timer to this wait so that a commit cancels it rather\n    // than leaving it to fire against a barrier that has already moved on.\n    const poll = new AbortController();\n    const abortPoll = () => poll.abort();\n    signal.addEventListener('abort', abortPoll, {once: true});\n    try {\n      const wokenBy = await Promise.race([\n        committed.promise.then(() => 'commit' as const),\n        this.#sleep(\n          Math.min(this.#barrierPollIntervalMs, remainingMs),\n          poll.signal,\n        ).then(() => 'poll' as const),\n      ]);\n      this.#barrierWakeups.add(1, {'wakeup.source': wokenBy});\n    } finally {\n      this.#commitWaiters.delete(waiter);\n      signal.removeEventListener('abort', abortPoll);\n      poll.abort();\n    }\n  }\n\n  #throwIfAborted(signal: AbortSignal): void {\n    if (this.#closed || signal.aborted) {\n      throw new AbortError('SQLite change-log catchup aborted');\n    }\n  }\n}\n\ntype CommitWaiter = {watermark: string; resolve: () => void};\n\n/**\n * A barrier that gave up on the replica reaching the required head. Both\n * reasons mean \"not yet\", not \"never\", so both end the subscription cleanly\n * for a retry rather than failing it.\n */\nexport class SQLiteChangeLogBarrierError extends Error {}\n\n/** The replica did not reach the required head before the deadline. */\nexport class SQLiteChangeLogBarrierTimeoutError extends SQLiteChangeLogBarrierError {\n  readonly name = 'SQLiteChangeLogBarrierTimeoutError';\n}\n\n/** Waiting any longer would have cost more than reconnecting. */\nexport class SQLiteChangeLogBarrierBacklogError extends SQLiteChangeLogBarrierError {\n  readonly name = 'SQLiteChangeLogBarrierBacklogError';\n}\n"],"mappings":";;;;;AAmCA,IAAM,mCAAmC;AAWzC,IAAM,qBAAkD,EACtD,uBAAsB,aAAY,QAAQ,QAAQ,SAAS,CAAC,EAC9D;;;;;;;;;;AAWA,IAAa,yBAAb,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,mBAA4B,mBAC1B,eACA,sCACA,0EACF;CACA,2BAAoC,mBAClC,eACA,+CACA,sIAEF;CACA,kBAA2B,mBACzB,eACA,qCACA,iLAGF;CACA,4BAAqB,IAAI,IAAiC;CAC1D,iCAA0B,IAAI,IAAkB;CAChD,UAAU;CAEV,YACE,IACA,WACA,QACA,MACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAKC,aAAa;EAClB,KAAKC,UAAU;EACf,KAAKC,aAAa,KAAK;EACvB,KAAKC,oBAAoB,KAAK;EAC9B,KAAKC,yBACH,KAAK,yBAAyB;EAChC,KAAKC,gBAAgB,KAAK,gBAAgB;EAC1C,KAAKC,SAAS,KAAK,SAAS;EAC5B,KAAKC,OAAO,KAAK,OAAO,KAAK;CAC/B;;;;;;CAOA,MAAM,QACJ,YACA,qBACe;EACf,IAAI,KAAKM,SAAS;GAChB,WAAW,KAAK,IAAI,WAAW,qCAAqC,CAAC;GACrE;EACF;EAEA,MAAM,QAAQ,IAAI,gBAAgB;EAClC,KAAKF,UAAU,IAAI,YAAY,KAAK;EACpC,IAAI;EACJ,IAAI;GACF,MAAM,KAAKN,cAAc,2BAA2B;IAClD,KAAKS,gBAAgB,MAAM,MAAM;IACjC,eAAe,oBAAoB;IACnC,KAAKd,WAAW,IAAI,UAAU;GAChC,CAAC;EACH,SAAS,OAAO;GACd,KAAKW,UAAU,OAAO,UAAU;GAChC,IAAI,CAAC,MAAM,OAAO,SAAS;IACzB,KAAKZ,IAAI,QACP,qDAAqD,WAAW,MAChE,KACF;IACA,WAAW,KAAK,KAAK;GACvB;GACA;EACF;EAEA,KAAUgB,KAAK,YAAY,cAA0C,KAAK;CAC5E;;;;;;;;;;CAWA,kBAAkB,WAAyB;EACzC,KAAK,MAAM,UAAU,KAAKH,gBACxB,IAAI,aAAa,OAAO,WAAW;GACjC,KAAKA,eAAe,OAAO,MAAM;GACjC,OAAO,QAAQ;EACjB;CAEJ;CAEA,OAAO,YAA8B;EACnC,KAAKD,UAAU,IAAI,UAAU,GAAG,MAAM;EACtC,KAAKA,UAAU,OAAO,UAAU;EAChC,KAAKX,WAAW,OAAO,UAAU;CACnC;CAEA,QAAc;EACZ,IAAI,KAAKa,SACP;EAEF,KAAKA,UAAU;EACf,KAAK,MAAM,CAAC,YAAY,UAAU,KAAKF,WAAW;GAChD,MAAM,MAAM;GACZ,KAAKX,WAAW,OAAO,UAAU;EACnC;EACA,KAAKW,UAAU,MAAM;EACrB,KAAKV,QAAQ,MAAM;CACrB;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,MAAM;CACb;CAEA,MAAMc,KACJ,YACA,cACA,OACe;EACf,MAAM,EAAC,WAAU;EACjB,IAAI;GACF,MAAM,oBAAoB,KAAKR,KAAK;GACpC,MAAM,WAAW,MAAM,KAAKS,mBAAmB,cAAc,MAAM;GACnE,MAAM,qBAAqB,KAAKT,KAAK,IAAI;GACzC,IAAI,qBAAqB,KAAKJ,mBAC5B,KAAKJ,IAAI,OACP,UAAU,mBAAmB,qFACsB,WAAW,IAChE;GAOF,MAAM,WAAW,KAAKQ,KAAK,IAAI,KAAKJ;GACpC,MAAM,OAAO,MAAM,KAAKc,aACtB,YACA,UACA,UACA,MACF;GACA,KAAKH,gBAAgB,MAAM;GAE3B,IAAI,KAAK,SAAS,WAAW;IAC3B,MAAM,UACJ,mCAAmC,KAAK,aAAa,cACvC,WAAW,UAAU;IACrC,KAAKf,IAAI,OACP,qCAAqC,WAAW,UAAU,wBAChC,KAAK,aAAa,EAC9C;IACA,WAAW,MAAM,GAA2B,OAAO;IACnD;GACF;GAEA,IAAI,QAAQ;GACZ,MAAM,QAAQ,KAAKQ,KAAK;GACxB,IAAI,KAAK,SAAS,SAAS;IACzB,IAAI;IACJ,WAAW,MAAM,WAAW,KAAKN,QAAQ,KACvC,WAAW,WACX,KAAK,eACL,KAAKC,YACL,MACF,GAAG;KACD,MAAM,YAAY,KAAKK,KAAK;KAC5B,MAAM;KACN,MAAM,UAAU,KAAKA,KAAK,IAAI;KAC9B,IAAI,mBACF,KAAKR,IAAI,UAAU,MAAM,SAAS,WAChC,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,WAAW,GAAG,8CAEvD;KAEF,KAAKe,gBAAgB,MAAM;KAC3B,KAAK,MAAM,UAAU,SAAS;MAC5B,oBAAoB,WAAW,QAAQ,MAAM;MAC7C;KACF;IACF;IACA,MAAM;GACR,OACE,KAAKf,IAAI,OACP,cAAc,WAAW,GAAG,gBAAgB,WAAW,UAAU,0CACrB,KAAK,cAAc,sCAEjE;GAGF,KAAKe,gBAAgB,MAAM;GAC3B,KAAKf,IAAI,OACP,aAAa,WAAW,GAAG,oBAAoB,MAAM,YAC/C,KAAKQ,KAAK,IAAI,MAAM,KAC5B;GAGA,WAAgB,YAAY;EAC9B,SAAS,OAAO;GACd,IAAI,OAAO,SACT;GAEF,IAAI,iBAAiB,6BAA6B;IAChD,IAAI,iBAAiB,oCACnB,KAAKC,iBAAiB,IAAI,CAAC;IAU7B,KAAKT,IAAI,OACP,2BAA2B,WAAW,GAAG,2BACzC,KACF;IACA,WAAW,MAAM;IACjB;GACF;GACA,KAAKA,IAAI,QACP,sCAAsC,WAAW,GAAG,eACpD,KACF;GACA,WAAW,KAAK,KAAK;EACvB,UAAU;GACR,IAAI,KAAKY,UAAU,IAAI,UAAU,MAAM,OACrC,KAAKA,UAAU,OAAO,UAAU;EAEpC;CACF;;;;;;;;;;CAWA,MAAMK,mBACJ,cACA,QACiB;EACjB,IAAI,OAAO,iBAAiB,UAC1B,OAAO;EAET,KAAKF,gBAAgB,MAAM;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,QAAQ,KAAK,CACxB,cACA,IAAI,SAAgB,GAAG,WAAW;IAChC,gBACE,OAAO,IAAI,WAAW,mCAAmC,CAAC;IAC5D,OAAO,iBAAiB,SAAS,SAAS,EAAC,MAAM,KAAI,CAAC;GACxD,CAAC,CACH,CAAC;EACH,UAAU;GACR,IAAI,SACF,OAAO,oBAAoB,SAAS,OAAO;EAE/C;CACF;CAEA,MAAMG,aACJ,YACA,cACA,UACA,QACoD;EAIpD,MAAM,cAAc,WAAW,gBAAgB;EAC/C,MAAM,UAAU,IAAI,gBAAgB;EACpC,MAAM,qBAAqB,QAAQ,MAAM;EACzC,OAAO,iBAAiB,SAAS,cAAc,EAAC,MAAM,KAAI,CAAC;EAC3D,IAAI,OAAO,SACT,QAAQ,MAAM;EAGhB,IAAI,eAAe,WAAW;EAC9B,IAAI;GACF,MAAM,aAAa,YAAY;IAC7B,OAAO,MAAM;KACX,KAAKH,gBAAgB,QAAQ,MAAM;KACnC,MAAM,OAAO,KAAKb,QAAQ,KAAK,WAAW,SAAS;KAOnD,IAAI,KAAK,SAAS,aAAa;MAC7B,eAAe,KAAK;MACpB,IAAI,KAAK,iBAAiB,cACxB,OAAO;KAEX;KACA,MAAM,YAAY,WAAW,KAAKM,KAAK;KACvC,IAAI,aAAa,GACf,MAAM,IAAI,mCACR,qCAAqC,aAAa,0BACzB,cAC3B;KAEF,MAAM,KAAKW,wBACT,cACA,WACA,QAAQ,MACV;IACF;GACF,GAAG;GAEH,MAAM,kBAAkB,YAAY,QAAQ,WAAW;IACrD,KAAKJ,gBAAgB,MAAM;IAG3B,IAAI,CAAC,WAAW,aACd,OAAO,IAAI,cAAqB,CAAC,CAAC;IAEpC,KAAKJ,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,UAAS,CAAC;IAIxD,KAAKD,yBAAyB,IAAI,CAAC;IACnC,MAAM,IAAI,mCACR,0BAA0B,WAAW,GAAG,6DACA,aAAa,0BAC5B,cAC3B;GACF,CAAC;GAED,OAAO,MAAM,QAAQ,KAAK,CAAC,WAAW,eAAe,CAAC;EACxD,UAAU;GACR,OAAO,oBAAoB,SAAS,YAAY;GAChD,QAAQ,MAAM;GACd,YAAY,OAAO;EACrB;CACF;;;;;;;;;CAUA,MAAMS,wBACJ,cACA,aACA,QACe;EACf,MAAM,YAAY,SAAe;EACjC,MAAM,SAAS;GAAC,WAAW;GAAc,SAAS,UAAU;EAAO;EACnE,KAAKN,eAAe,IAAI,MAAM;EAG9B,MAAM,OAAO,IAAI,gBAAgB;EACjC,MAAM,kBAAkB,KAAK,MAAM;EACnC,OAAO,iBAAiB,SAAS,WAAW,EAAC,MAAM,KAAI,CAAC;EACxD,IAAI;GACF,MAAM,UAAU,MAAM,QAAQ,KAAK,CACjC,UAAU,QAAQ,WAAW,QAAiB,GAC9C,KAAKN,OACH,KAAK,IAAI,KAAKF,wBAAwB,WAAW,GACjD,KAAK,MACP,EAAE,WAAW,MAAe,CAC9B,CAAC;GACD,KAAKM,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,QAAO,CAAC;EACxD,UAAU;GACR,KAAKE,eAAe,OAAO,MAAM;GACjC,OAAO,oBAAoB,SAAS,SAAS;GAC7C,KAAK,MAAM;EACb;CACF;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAKC,WAAW,OAAO,SACzB,MAAM,IAAI,WAAW,mCAAmC;CAE5D;AACF;;;;;;AASA,IAAa,8BAAb,cAAiD,MAAM,CAAC;;AAGxD,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB;;AAGA,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB"}
+{"version":3,"file":"sqlite-change-log-catchup.js","names":["#lc","#forwarder","#reader","#batchSize","#barrierTimeoutMs","#barrierPollIntervalMs","#cleanupGuard","#sleep","#now","#onFailure","#barrierTimeouts","#barrierBacklogOverflows","#barrierWakeups","#catchupResults","#catchupRows","#catchupBytes","#catchupDuration","#barrierWaitDuration","#catchupBacklogPeak","#catchups","#commitWaiters","#closed","#throwIfAborted","#run","#awaitRequiredHead","#waitForPlan","#awaitChangeLogProgress"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateLatencyHistogram,\n  getOrCreateValueHistogram,\n} from '../../observability/metrics.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {Forwarder} from './forwarder.ts';\nimport type {CatchupPlan} from './sqlite-change-log-reader.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport interface SQLiteChangeLogCatchupReader {\n  plan(fromWatermark: string): CatchupPlan;\n  read(\n    fromWatermark: string,\n    throughWatermark: string,\n    batchSize: number,\n    signal?: AbortSignal,\n  ): AsyncIterable<readonly WatermarkedChange[]>;\n  close(): void;\n}\n\n/**\n * Slice 9 replaces this no-op implementation with a guard that waits for an\n * in-flight purge and blocks new purge dispatch while the subscriber's ACK is\n * made visible to the Forwarder.\n */\nexport interface SQLiteChangeLogCleanupGuard {\n  runWhilePurgeBlocked<T>(register: () => T): Promise<T>;\n}\n\n// How long the barrier waits for the change-log writer's commit notification\n// before re-reading the log itself. The notification normally arrives first, so\n// this is a backstop for the windows in which none is coming: the writer is\n// disabled, or it is between stream connections.\nconst DEFAULT_BARRIER_POLL_INTERVAL_MS = 1000;\n\nexport type SQLiteChangeLogCatchupOptions = {\n  batchSize: number;\n  barrierTimeoutMs: number;\n  barrierPollIntervalMs?: number | undefined;\n  cleanupGuard?: SQLiteChangeLogCleanupGuard | undefined;\n  sleep?: typeof sleep | undefined;\n  now?: (() => number) | undefined;\n  /** Opens slice 11's process-local breaker after registration has committed. */\n  onFailure?: ((failure: SQLiteChangeLogCatchupFailure) => void) | undefined;\n};\n\nexport type SQLiteChangeLogCatchupRequest = {\n  /**\n   * Whether the routing warm-up gate classified the log as warm *for this\n   * subscriber*. The coordinator is opened once and reused, so this cannot be\n   * a property of it: `coldReadPercent` serves some subscribers from a log\n   * that is still inside its warm-up window, and the catchup metrics have to\n   * separate those from steady-state serving.\n   */\n  readonly logWarm?: boolean | undefined;\n};\n\nexport type SQLiteChangeLogCatchupFailure = 'barrier-timeout' | 'reader-error';\n\nexport type SQLiteChangeLogCatchupRegistration =\n  | {readonly kind: 'registered'}\n  | {readonly kind: 'uncovered'; readonly minWatermark: string}\n  // Registration failed before the subscriber was committed to SQLite, so PG\n  // catchup is still available and the caller falls back to it.\n  | {readonly kind: 'declined'; readonly error: unknown}\n  // The coordinator closed or failed the subscriber itself.\n  | {readonly kind: 'handled'};\n\nconst NOOP_CLEANUP_GUARD: SQLiteChangeLogCleanupGuard = {\n  runWhilePurgeBlocked: register => Promise.resolve(register()),\n};\n\n/**\n * Coordinates a serving subscriber's gap-free transition from replica-local\n * SQLite catchup to the Forwarder's live stream.\n *\n * Registration and required-head capture happen in one synchronous callback.\n * The subscriber therefore either sees a transaction in SQLite catchup or in\n * its live backlog (duplicates across that boundary are filtered by\n * Subscriber), never in neither place.\n */\nexport class SQLiteChangeLogCatchup implements Disposable {\n  readonly #lc: LogContext;\n  readonly #forwarder: Forwarder;\n  readonly #reader: SQLiteChangeLogCatchupReader;\n  readonly #batchSize: number;\n  readonly #barrierTimeoutMs: number;\n  readonly #barrierPollIntervalMs: number;\n  readonly #cleanupGuard: SQLiteChangeLogCleanupGuard;\n  readonly #sleep: typeof sleep;\n  readonly #now: () => number;\n  readonly #onFailure:\n    | ((failure: SQLiteChangeLogCatchupFailure) => void)\n    | undefined;\n  readonly #barrierTimeouts = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_timeouts',\n    'SQLite change-log catchups that timed out waiting for the required head.',\n  );\n  readonly #barrierBacklogOverflows = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_backlog_overflows',\n    'SQLite change-log catchups abandoned because the subscriber backlog ' +\n      'reached its high water mark while waiting for the required head.',\n  );\n  readonly #barrierWakeups = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_wakeups',\n    'SQLite catchup barrier waits, labeled by what ended the wait. A ' +\n      'population dominated by \"poll\" means the change-log writer\\'s commit ' +\n      'notification is not reaching the barrier.',\n  );\n  readonly #catchupResults = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.catchup_results',\n    'SQLite catchup subscriptions by outcome.',\n  );\n  readonly #catchupRows = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.catchup_rows',\n    'Rows served from the SQLite change log during catchup.',\n  );\n  readonly #catchupBytes = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.catchup_bytes',\n    {\n      description: 'Bytes served from the SQLite change log during catchup.',\n      unit: 'By',\n    },\n  );\n  readonly #catchupDuration = getOrCreateLatencyHistogram(\n    'replication',\n    'sqlite_change_log.catchup_duration',\n    'Duration of a SQLite change-log catchup.',\n  );\n  readonly #barrierWaitDuration = getOrCreateLatencyHistogram(\n    'replication',\n    'sqlite_change_log.barrier_wait_duration',\n    'Time SQLite catchup waits for its pinned required head.',\n  );\n  readonly #catchupBacklogPeak = getOrCreateValueHistogram(\n    'replication',\n    'sqlite_change_log.catchup_backlog_peak_bytes',\n    {\n      description:\n        'Peak live backlog bytes buffered while a subscriber catches up from SQLite.',\n      unit: 'By',\n      bucketBoundaries: [\n        0, 1024, 16_384, 65_536, 262_144, 1_048_576, 4_194_304, 16_777_216,\n      ],\n    },\n  );\n  readonly #catchups = new Map<Subscriber, AbortController>();\n  readonly #commitWaiters = new Set<CommitWaiter>();\n  #closed = false;\n\n  constructor(\n    lc: LogContext,\n    forwarder: Forwarder,\n    reader: SQLiteChangeLogCatchupReader,\n    opts: SQLiteChangeLogCatchupOptions,\n  ) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-catchup');\n    this.#forwarder = forwarder;\n    this.#reader = reader;\n    this.#batchSize = opts.batchSize;\n    this.#barrierTimeoutMs = opts.barrierTimeoutMs;\n    this.#barrierPollIntervalMs =\n      opts.barrierPollIntervalMs ?? DEFAULT_BARRIER_POLL_INTERVAL_MS;\n    this.#cleanupGuard = opts.cleanupGuard ?? NOOP_CLEANUP_GUARD;\n    this.#sleep = opts.sleep ?? sleep;\n    this.#now = opts.now ?? Date.now;\n    this.#onFailure = opts.onFailure;\n  }\n\n  /**\n   * Registers the subscriber before resolving. Catchup itself continues in\n   * the background so subscribe() does not wait for SQLite to reach the\n   * required head.\n   */\n  async catchup(\n    subscriber: Subscriber,\n    captureRequiredHead: () => string | Promise<string>,\n    request: SQLiteChangeLogCatchupRequest = {},\n  ): Promise<SQLiteChangeLogCatchupRegistration> {\n    if (this.#closed) {\n      subscriber.fail(new AbortError('SQLite change-log catchup is closed'));\n      return {kind: 'handled'};\n    }\n\n    const abort = new AbortController();\n    this.#catchups.set(subscriber, abort);\n    let requiredHead: string | Promise<string> | undefined;\n    let uncoveredMinWatermark: string | undefined;\n    let committed = false;\n    try {\n      await this.#cleanupGuard.runWhilePurgeBlocked(() => {\n        this.#throwIfAborted(abort.signal);\n        // The router's eligibility inspection happens before this async guard\n        // is acquired. Recheck the exact boundary here: an in-flight purge can\n        // have advanced the minimum while registration waited. Once added to\n        // the Forwarder, the subscriber's ACK prevents any later purge from\n        // crossing this boundary.\n        const plan = this.#reader.plan(subscriber.watermark);\n        if (plan.kind === 'too-old') {\n          uncoveredMinWatermark = plan.minWatermark;\n          return;\n        }\n        requiredHead = captureRequiredHead();\n        this.#forwarder.add(subscriber);\n        committed = true;\n      });\n    } catch (error) {\n      this.#catchups.delete(subscriber);\n      if (abort.signal.aborted) {\n        return {kind: 'handled'};\n      }\n      if (!committed) {\n        // Nothing was committed to SQLite -- the subscriber never reached\n        // Forwarder.add() -- so PG catchup is still available. Failing here\n        // would send the client a terminal ['error', ...], which\n        // IncrementalSyncer answers with a full litestream replica restore,\n        // for a fallback that was still open. Trip the breaker as well: this\n        // is a reader failure like any other, and without it the retry picks\n        // SQLite again and the reset repeats.\n        this.#onFailure?.('reader-error');\n        return {kind: 'declined', error};\n      }\n      this.#lc.error?.(\n        `error while registering SQLite catchup subscriber ${subscriber.id}`,\n        error,\n      );\n      subscriber.fail(error);\n      return {kind: 'handled'};\n    }\n    if (uncoveredMinWatermark !== undefined) {\n      this.#catchups.delete(subscriber);\n      return {kind: 'uncovered', minWatermark: uncoveredMinWatermark};\n    }\n\n    void this.#run(\n      subscriber,\n      requiredHead as string | Promise<string>,\n      abort,\n      request.logWarm,\n    );\n    return {kind: 'registered'};\n  }\n\n  /**\n   * Notes that the change-log writer has committed `watermark`, i.e. that the\n   * log's head has advanced to it. The writer runs in this process, so this is\n   * the event itself rather than a subscriber's ACK of it.\n   *\n   * It is a wakeup, not an authority: `plan()` still decides what can be read.\n   * A notification that never arrives costs the barrier a poll interval, and one\n   * that arrives early costs an extra `plan()` call.\n   */\n  onChangeLogCommit(watermark: string): void {\n    for (const waiter of this.#commitWaiters) {\n      if (watermark >= waiter.watermark) {\n        this.#commitWaiters.delete(waiter);\n        waiter.resolve();\n      }\n    }\n  }\n\n  remove(subscriber: Subscriber): void {\n    this.#catchups.get(subscriber)?.abort();\n    this.#catchups.delete(subscriber);\n    this.#forwarder.remove(subscriber);\n  }\n\n  close(): void {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    for (const [subscriber, abort] of this.#catchups) {\n      abort.abort();\n      this.#forwarder.remove(subscriber);\n    }\n    this.#catchups.clear();\n    this.#reader.close();\n  }\n\n  [Symbol.dispose](): void {\n    this.close();\n  }\n\n  async #run(\n    subscriber: Subscriber,\n    requiredHead: string | Promise<string>,\n    abort: AbortController,\n    logWarm: boolean | undefined,\n  ): Promise<void> {\n    const {signal} = abort;\n    const catchupStart = this.#now();\n    let outcome = 'aborted';\n    let backlogPeakBytes = 0;\n    try {\n      const requiredHeadStart = this.#now();\n      const required = await this.#awaitRequiredHead(requiredHead, signal);\n      const requiredHeadWaitMs = this.#now() - requiredHeadStart;\n      if (requiredHeadWaitMs > this.#barrierTimeoutMs) {\n        this.#lc.info?.(\n          `waited ${requiredHeadWaitMs} ms for the forwarded transaction to ` +\n            `commit before starting the SQLite barrier for ${subscriber.id}`,\n        );\n      }\n      // The barrier bounds replica lag, not upstream transaction duration, so\n      // its deadline starts once the required head is known. Sharing one\n      // deadline with the wait above would fail subscribers that registered\n      // during a transaction longer than barrierTimeoutMs, however current\n      // the replica is.\n      const deadline = this.#now() + this.#barrierTimeoutMs;\n      const barrierStart = this.#now();\n      let plan: CatchupPlan;\n      try {\n        plan = await this.#waitForPlan(subscriber, required, deadline, signal);\n      } finally {\n        // Record failed and aborted barriers too; their wait distribution is\n        // at least as operationally important as the successful path.\n        this.#barrierWaitDuration.recordMs(this.#now() - barrierStart);\n      }\n      this.#throwIfAborted(signal);\n\n      if (plan.kind === 'too-old') {\n        outcome = 'too-old';\n        const message =\n          `earliest supported watermark is ${plan.minWatermark} ` +\n          `(requested ${subscriber.watermark})`;\n        this.#lc.warn?.(\n          `rejecting subscriber at watermark ${subscriber.watermark} ` +\n            `(earliest watermark: ${plan.minWatermark})`,\n        );\n        subscriber.close(ErrorType.WatermarkTooOld, message);\n        return;\n      }\n\n      let count = 0;\n      let bytes = 0;\n      const start = this.#now();\n      if (plan.kind === 'range') {\n        let lastBatchConsumed: Promise<unknown> | undefined;\n        for await (const changes of this.#reader.read(\n          subscriber.watermark,\n          plan.headWatermark,\n          this.#batchSize,\n          signal,\n        )) {\n          const waitStart = this.#now();\n          await lastBatchConsumed;\n          const elapsed = this.#now() - waitStart;\n          if (lastBatchConsumed) {\n            this.#lc[elapsed > 100 ? 'info' : 'debug']?.(\n              `waited ${elapsed.toFixed(3)} ms for ${subscriber.id} to consume ` +\n                `the previous SQLite catchup batch`,\n            );\n          }\n          this.#throwIfAborted(signal);\n          for (const change of changes) {\n            lastBatchConsumed = subscriber.catchup(change);\n            count++;\n            bytes += Buffer.byteLength(change[2]);\n          }\n          backlogPeakBytes = Math.max(\n            backlogPeakBytes,\n            subscriber.getStats().backlogBytes,\n          );\n        }\n        await lastBatchConsumed;\n        outcome = 'range';\n      } else {\n        outcome = 'ahead';\n        this.#lc.warn?.(\n          `subscriber ${subscriber.id} at watermark ${subscriber.watermark} ` +\n            `is ahead of the SQLite change-log head ${plan.headWatermark}; ` +\n            `waiting for the replica to catch up`,\n        );\n      }\n\n      this.#throwIfAborted(signal);\n      this.#lc.info?.(\n        `caught up ${subscriber.id} from SQLite with ${count} changes ` +\n          `(${this.#now() - start} ms)`,\n      );\n      const attributes = {\n        classification: outcome,\n        log_warm: logWarm ?? 'unknown',\n      };\n      this.#catchupRows.add(count, attributes);\n      this.#catchupBytes.add(bytes, attributes);\n      // Keep buffering live sends until the asynchronous backlog drain has\n      // established its ordering boundary.\n      void subscriber.setCaughtUp();\n    } catch (error) {\n      if (signal.aborted) {\n        return;\n      }\n      if (error instanceof SQLiteChangeLogBarrierError) {\n        if (error instanceof SQLiteChangeLogBarrierTimeoutError) {\n          this.#barrierTimeouts.add(1);\n          outcome = 'barrier-timeout';\n          this.#onFailure?.('barrier-timeout');\n        } else {\n          // No #onFailure: a backlog high water mark is a property of this\n          // one subscriber's consumption, not of the log. Tripping the\n          // process-wide breaker for it would take every task in the shard\n          // off SQLite, which a client that reconnects and stalls again\n          // could hold open indefinitely.\n          outcome = 'barrier-backlog';\n        }\n        // Giving up on the barrier means the replica has not caught up *yet*,\n        // not that it cannot serve this subscriber. End the subscription\n        // cleanly so the client reconnects and retries: IncrementalSyncer\n        // treats any ['error', ...] message as terminal and restores a fresh\n        // replica from litestream, whereas a clean end backs off and\n        // re-subscribes. If retries keep failing until the change log is purged\n        // past the subscriber's watermark, plan() returns 'too-old', which is\n        // terminal by design.\n        this.#lc.warn?.(\n          `ending subscription for ${subscriber.id} to retry SQLite catchup`,\n          error,\n        );\n        subscriber.close();\n        return;\n      }\n      outcome = 'reader-error';\n      this.#onFailure?.('reader-error');\n      this.#lc.error?.(\n        `error while catching up subscriber ${subscriber.id} from SQLite`,\n        error,\n      );\n      subscriber.fail(error);\n    } finally {\n      backlogPeakBytes = Math.max(\n        backlogPeakBytes,\n        subscriber.getStats().backlogBytes,\n      );\n      const attributes = {\n        classification: outcome,\n        log_warm: logWarm ?? 'unknown',\n      };\n      this.#catchupResults.add(1, attributes);\n      this.#catchupDuration.recordMs(this.#now() - catchupStart, attributes);\n      this.#catchupBacklogPeak.record(backlogPeakBytes, attributes);\n      if (this.#catchups.get(subscriber) === abort) {\n        this.#catchups.delete(subscriber);\n      }\n    }\n  }\n\n  /**\n   * Waits for the transaction that was in flight at registration to finish.\n   *\n   * This wait is bounded by the upstream transaction rather than by\n   * `barrierTimeoutMs`: the completion settles on both commit and rollback,\n   * and an interrupted change stream forwards a synthetic rollback, so it\n   * always settles. Aborting -- the subscriber disconnecting or the service\n   * shutting down -- is what releases it early.\n   */\n  async #awaitRequiredHead(\n    requiredHead: string | Promise<string>,\n    signal: AbortSignal,\n  ): Promise<string> {\n    if (typeof requiredHead === 'string') {\n      return requiredHead;\n    }\n    this.#throwIfAborted(signal);\n    let onAbort: (() => void) | undefined;\n    try {\n      return await Promise.race([\n        requiredHead,\n        new Promise<never>((_, reject) => {\n          onAbort = () =>\n            reject(new AbortError('SQLite change-log catchup aborted'));\n          signal.addEventListener('abort', onAbort, {once: true});\n        }),\n      ]);\n    } finally {\n      if (onAbort) {\n        signal.removeEventListener('abort', onAbort);\n      }\n    }\n  }\n\n  async #waitForPlan(\n    subscriber: Subscriber,\n    requiredHead: string,\n    deadline: number,\n    signal: AbortSignal,\n  ): Promise<Exclude<CatchupPlan, {kind: 'not-ready'}>> {\n    // Race backlog growth once around the whole polling loop. Attaching the\n    // same unresolved promise to every per-poll race would retain one reaction\n    // per iteration until the subscriber's backlog eventually filled.\n    const backlogFull = subscriber.whenBacklogFull();\n    const barrier = new AbortController();\n    const abortBarrier = () => barrier.abort();\n    signal.addEventListener('abort', abortBarrier, {once: true});\n    if (signal.aborted) {\n      barrier.abort();\n    }\n\n    let observedHead = subscriber.watermark;\n    try {\n      const planReady = (async () => {\n        while (true) {\n          this.#throwIfAborted(barrier.signal);\n          const plan = this.#reader.plan(subscriber.watermark);\n          // A not-ready log has no head to compare, so it can never satisfy\n          // the required head: keep waiting for the writer to create and\n          // reconcile it, and let the barrier deadline end the subscription\n          // cleanly for a retry. Selection declines a not-ready log before a\n          // subscriber gets here, and neither purging nor reconciliation can\n          // empty a log that had content, so this is a belt-and-braces path.\n          if (plan.kind !== 'not-ready') {\n            observedHead = plan.headWatermark;\n            if (plan.headWatermark >= requiredHead) {\n              return plan;\n            }\n          }\n          const remaining = deadline - this.#now();\n          if (remaining <= 0) {\n            throw new SQLiteChangeLogBarrierTimeoutError(\n              `timed out waiting for SQLite head ${observedHead} to ` +\n                `reach required head ${requiredHead}`,\n            );\n          }\n          await this.#awaitChangeLogProgress(\n            requiredHead,\n            remaining,\n            barrier.signal,\n          );\n        }\n      })();\n\n      const backlogOverflow = backlogFull.promise.then(() => {\n        this.#throwIfAborted(signal);\n        // close() also resolves backlog waiters so none are stranded. The\n        // backlog is cleared first, which distinguishes close from overflow.\n        if (!subscriber.backlogFull) {\n          return new Promise<never>(() => {});\n        }\n        this.#barrierWakeups.add(1, {'wakeup.source': 'backlog'});\n        // Bytes, not time, are what the barrier actually costs: past the high\n        // water mark this subscriber's send() no longer resolves, so it holds\n        // up every flush and can stall the replica this barrier waits on.\n        this.#barrierBacklogOverflows.add(1);\n        throw new SQLiteChangeLogBarrierBacklogError(\n          `backlog for subscriber ${subscriber.id} reached its high water ` +\n            `mark while waiting for SQLite head ${observedHead} to ` +\n            `reach required head ${requiredHead}`,\n        );\n      });\n\n      return await Promise.race([planReady, backlogOverflow]);\n    } finally {\n      signal.removeEventListener('abort', abortBarrier);\n      barrier.abort();\n      backlogFull.cancel();\n    }\n  }\n\n  /**\n   * Waits for the change-log writer to commit `requiredHead`, or the poll\n   * interval as a backstop.\n   *\n   * No notification can be missed by registering after `plan()` was read: the\n   * notification follows the commit it reports, so if the writer had already\n   * committed `requiredHead`, `plan()` would have seen it.\n   */\n  async #awaitChangeLogProgress(\n    requiredHead: string,\n    remainingMs: number,\n    signal: AbortSignal,\n  ): Promise<void> {\n    const committed = resolver<void>();\n    const waiter = {watermark: requiredHead, resolve: committed.resolve};\n    this.#commitWaiters.add(waiter);\n    // Scopes the backstop timer to this wait so that a commit cancels it rather\n    // than leaving it to fire against a barrier that has already moved on.\n    const poll = new AbortController();\n    const abortPoll = () => poll.abort();\n    signal.addEventListener('abort', abortPoll, {once: true});\n    try {\n      const wokenBy = await Promise.race([\n        committed.promise.then(() => 'commit' as const),\n        this.#sleep(\n          Math.min(this.#barrierPollIntervalMs, remainingMs),\n          poll.signal,\n        ).then(() => 'poll' as const),\n      ]);\n      this.#barrierWakeups.add(1, {'wakeup.source': wokenBy});\n    } finally {\n      this.#commitWaiters.delete(waiter);\n      signal.removeEventListener('abort', abortPoll);\n      poll.abort();\n    }\n  }\n\n  #throwIfAborted(signal: AbortSignal): void {\n    if (this.#closed || signal.aborted) {\n      throw new AbortError('SQLite change-log catchup aborted');\n    }\n  }\n}\n\ntype CommitWaiter = {watermark: string; resolve: () => void};\n\n/**\n * A barrier that gave up on the replica reaching the required head. Both\n * reasons mean \"not yet\", not \"never\", so both end the subscription cleanly\n * for a retry rather than failing it.\n */\nexport class SQLiteChangeLogBarrierError extends Error {}\n\n/** The replica did not reach the required head before the deadline. */\nexport class SQLiteChangeLogBarrierTimeoutError extends SQLiteChangeLogBarrierError {\n  readonly name = 'SQLiteChangeLogBarrierTimeoutError';\n}\n\n/** Waiting any longer would have cost more than reconnecting. */\nexport class SQLiteChangeLogBarrierBacklogError extends SQLiteChangeLogBarrierError {\n  readonly name = 'SQLiteChangeLogBarrierBacklogError';\n}\n"],"mappings":";;;;;AAuCA,IAAM,mCAAmC;AAmCzC,IAAM,qBAAkD,EACtD,uBAAsB,aAAY,QAAQ,QAAQ,SAAS,CAAC,EAC9D;;;;;;;;;;AAWA,IAAa,yBAAb,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA,mBAA4B,mBAC1B,eACA,sCACA,0EACF;CACA,2BAAoC,mBAClC,eACA,+CACA,sIAEF;CACA,kBAA2B,mBACzB,eACA,qCACA,iLAGF;CACA,kBAA2B,mBACzB,eACA,qCACA,0CACF;CACA,eAAwB,mBACtB,eACA,kCACA,wDACF;CACA,gBAAyB,mBACvB,eACA,mCACA;EACE,aAAa;EACb,MAAM;CACR,CACF;CACA,mBAA4B,4BAC1B,eACA,sCACA,0CACF;CACA,uBAAgC,4BAC9B,eACA,2CACA,yDACF;CACA,sBAA+B,0BAC7B,eACA,gDACA;EACE,aACE;EACF,MAAM;EACN,kBAAkB;GAChB;GAAG;GAAM;GAAQ;GAAQ;GAAS;GAAW;GAAW;EAC1D;CACF,CACF;CACA,4BAAqB,IAAI,IAAiC;CAC1D,iCAA0B,IAAI,IAAkB;CAChD,UAAU;CAEV,YACE,IACA,WACA,QACA,MACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAKC,aAAa;EAClB,KAAKC,UAAU;EACf,KAAKC,aAAa,KAAK;EACvB,KAAKC,oBAAoB,KAAK;EAC9B,KAAKC,yBACH,KAAK,yBAAyB;EAChC,KAAKC,gBAAgB,KAAK,gBAAgB;EAC1C,KAAKC,SAAS,KAAK,SAAS;EAC5B,KAAKC,OAAO,KAAK,OAAO,KAAK;EAC7B,KAAKC,aAAa,KAAK;CACzB;;;;;;CAOA,MAAM,QACJ,YACA,qBACA,UAAyC,CAAC,GACG;EAC7C,IAAI,KAAKY,SAAS;GAChB,WAAW,KAAK,IAAI,WAAW,qCAAqC,CAAC;GACrE,OAAO,EAAC,MAAM,UAAS;EACzB;EAEA,MAAM,QAAQ,IAAI,gBAAgB;EAClC,KAAKF,UAAU,IAAI,YAAY,KAAK;EACpC,IAAI;EACJ,IAAI;EACJ,IAAI,YAAY;EAChB,IAAI;GACF,MAAM,KAAKb,cAAc,2BAA2B;IAClD,KAAKgB,gBAAgB,MAAM,MAAM;IAMjC,MAAM,OAAO,KAAKpB,QAAQ,KAAK,WAAW,SAAS;IACnD,IAAI,KAAK,SAAS,WAAW;KAC3B,wBAAwB,KAAK;KAC7B;IACF;IACA,eAAe,oBAAoB;IACnC,KAAKD,WAAW,IAAI,UAAU;IAC9B,YAAY;GACd,CAAC;EACH,SAAS,OAAO;GACd,KAAKkB,UAAU,OAAO,UAAU;GAChC,IAAI,MAAM,OAAO,SACf,OAAO,EAAC,MAAM,UAAS;GAEzB,IAAI,CAAC,WAAW;IAQd,KAAKV,aAAa,cAAc;IAChC,OAAO;KAAC,MAAM;KAAY;IAAK;GACjC;GACA,KAAKT,IAAI,QACP,qDAAqD,WAAW,MAChE,KACF;GACA,WAAW,KAAK,KAAK;GACrB,OAAO,EAAC,MAAM,UAAS;EACzB;EACA,IAAI,0BAA0B,KAAA,GAAW;GACvC,KAAKmB,UAAU,OAAO,UAAU;GAChC,OAAO;IAAC,MAAM;IAAa,cAAc;GAAqB;EAChE;EAEA,KAAUI,KACR,YACA,cACA,OACA,QAAQ,OACV;EACA,OAAO,EAAC,MAAM,aAAY;CAC5B;;;;;;;;;;CAWA,kBAAkB,WAAyB;EACzC,KAAK,MAAM,UAAU,KAAKH,gBACxB,IAAI,aAAa,OAAO,WAAW;GACjC,KAAKA,eAAe,OAAO,MAAM;GACjC,OAAO,QAAQ;EACjB;CAEJ;CAEA,OAAO,YAA8B;EACnC,KAAKD,UAAU,IAAI,UAAU,GAAG,MAAM;EACtC,KAAKA,UAAU,OAAO,UAAU;EAChC,KAAKlB,WAAW,OAAO,UAAU;CACnC;CAEA,QAAc;EACZ,IAAI,KAAKoB,SACP;EAEF,KAAKA,UAAU;EACf,KAAK,MAAM,CAAC,YAAY,UAAU,KAAKF,WAAW;GAChD,MAAM,MAAM;GACZ,KAAKlB,WAAW,OAAO,UAAU;EACnC;EACA,KAAKkB,UAAU,MAAM;EACrB,KAAKjB,QAAQ,MAAM;CACrB;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,MAAM;CACb;CAEA,MAAMqB,KACJ,YACA,cACA,OACA,SACe;EACf,MAAM,EAAC,WAAU;EACjB,MAAM,eAAe,KAAKf,KAAK;EAC/B,IAAI,UAAU;EACd,IAAI,mBAAmB;EACvB,IAAI;GACF,MAAM,oBAAoB,KAAKA,KAAK;GACpC,MAAM,WAAW,MAAM,KAAKgB,mBAAmB,cAAc,MAAM;GACnE,MAAM,qBAAqB,KAAKhB,KAAK,IAAI;GACzC,IAAI,qBAAqB,KAAKJ,mBAC5B,KAAKJ,IAAI,OACP,UAAU,mBAAmB,qFACsB,WAAW,IAChE;GAOF,MAAM,WAAW,KAAKQ,KAAK,IAAI,KAAKJ;GACpC,MAAM,eAAe,KAAKI,KAAK;GAC/B,IAAI;GACJ,IAAI;IACF,OAAO,MAAM,KAAKiB,aAAa,YAAY,UAAU,UAAU,MAAM;GACvE,UAAU;IAGR,KAAKR,qBAAqB,SAAS,KAAKT,KAAK,IAAI,YAAY;GAC/D;GACA,KAAKc,gBAAgB,MAAM;GAE3B,IAAI,KAAK,SAAS,WAAW;IAC3B,UAAU;IACV,MAAM,UACJ,mCAAmC,KAAK,aAAa,cACvC,WAAW,UAAU;IACrC,KAAKtB,IAAI,OACP,qCAAqC,WAAW,UAAU,wBAChC,KAAK,aAAa,EAC9C;IACA,WAAW,MAAM,GAA2B,OAAO;IACnD;GACF;GAEA,IAAI,QAAQ;GACZ,IAAI,QAAQ;GACZ,MAAM,QAAQ,KAAKQ,KAAK;GACxB,IAAI,KAAK,SAAS,SAAS;IACzB,IAAI;IACJ,WAAW,MAAM,WAAW,KAAKN,QAAQ,KACvC,WAAW,WACX,KAAK,eACL,KAAKC,YACL,MACF,GAAG;KACD,MAAM,YAAY,KAAKK,KAAK;KAC5B,MAAM;KACN,MAAM,UAAU,KAAKA,KAAK,IAAI;KAC9B,IAAI,mBACF,KAAKR,IAAI,UAAU,MAAM,SAAS,WAChC,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,WAAW,GAAG,8CAEvD;KAEF,KAAKsB,gBAAgB,MAAM;KAC3B,KAAK,MAAM,UAAU,SAAS;MAC5B,oBAAoB,WAAW,QAAQ,MAAM;MAC7C;MACA,SAAS,OAAO,WAAW,OAAO,EAAE;KACtC;KACA,mBAAmB,KAAK,IACtB,kBACA,WAAW,SAAS,EAAE,YACxB;IACF;IACA,MAAM;IACN,UAAU;GACZ,OAAO;IACL,UAAU;IACV,KAAKtB,IAAI,OACP,cAAc,WAAW,GAAG,gBAAgB,WAAW,UAAU,0CACrB,KAAK,cAAc,sCAEjE;GACF;GAEA,KAAKsB,gBAAgB,MAAM;GAC3B,KAAKtB,IAAI,OACP,aAAa,WAAW,GAAG,oBAAoB,MAAM,YAC/C,KAAKQ,KAAK,IAAI,MAAM,KAC5B;GACA,MAAM,aAAa;IACjB,gBAAgB;IAChB,UAAU,WAAW;GACvB;GACA,KAAKM,aAAa,IAAI,OAAO,UAAU;GACvC,KAAKC,cAAc,IAAI,OAAO,UAAU;GAGxC,WAAgB,YAAY;EAC9B,SAAS,OAAO;GACd,IAAI,OAAO,SACT;GAEF,IAAI,iBAAiB,6BAA6B;IAChD,IAAI,iBAAiB,oCAAoC;KACvD,KAAKL,iBAAiB,IAAI,CAAC;KAC3B,UAAU;KACV,KAAKD,aAAa,iBAAiB;IACrC,OAME,UAAU;IAUZ,KAAKT,IAAI,OACP,2BAA2B,WAAW,GAAG,2BACzC,KACF;IACA,WAAW,MAAM;IACjB;GACF;GACA,UAAU;GACV,KAAKS,aAAa,cAAc;GAChC,KAAKT,IAAI,QACP,sCAAsC,WAAW,GAAG,eACpD,KACF;GACA,WAAW,KAAK,KAAK;EACvB,UAAU;GACR,mBAAmB,KAAK,IACtB,kBACA,WAAW,SAAS,EAAE,YACxB;GACA,MAAM,aAAa;IACjB,gBAAgB;IAChB,UAAU,WAAW;GACvB;GACA,KAAKa,gBAAgB,IAAI,GAAG,UAAU;GACtC,KAAKG,iBAAiB,SAAS,KAAKR,KAAK,IAAI,cAAc,UAAU;GACrE,KAAKU,oBAAoB,OAAO,kBAAkB,UAAU;GAC5D,IAAI,KAAKC,UAAU,IAAI,UAAU,MAAM,OACrC,KAAKA,UAAU,OAAO,UAAU;EAEpC;CACF;;;;;;;;;;CAWA,MAAMK,mBACJ,cACA,QACiB;EACjB,IAAI,OAAO,iBAAiB,UAC1B,OAAO;EAET,KAAKF,gBAAgB,MAAM;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,QAAQ,KAAK,CACxB,cACA,IAAI,SAAgB,GAAG,WAAW;IAChC,gBACE,OAAO,IAAI,WAAW,mCAAmC,CAAC;IAC5D,OAAO,iBAAiB,SAAS,SAAS,EAAC,MAAM,KAAI,CAAC;GACxD,CAAC,CACH,CAAC;EACH,UAAU;GACR,IAAI,SACF,OAAO,oBAAoB,SAAS,OAAO;EAE/C;CACF;CAEA,MAAMG,aACJ,YACA,cACA,UACA,QACoD;EAIpD,MAAM,cAAc,WAAW,gBAAgB;EAC/C,MAAM,UAAU,IAAI,gBAAgB;EACpC,MAAM,qBAAqB,QAAQ,MAAM;EACzC,OAAO,iBAAiB,SAAS,cAAc,EAAC,MAAM,KAAI,CAAC;EAC3D,IAAI,OAAO,SACT,QAAQ,MAAM;EAGhB,IAAI,eAAe,WAAW;EAC9B,IAAI;GACF,MAAM,aAAa,YAAY;IAC7B,OAAO,MAAM;KACX,KAAKH,gBAAgB,QAAQ,MAAM;KACnC,MAAM,OAAO,KAAKpB,QAAQ,KAAK,WAAW,SAAS;KAOnD,IAAI,KAAK,SAAS,aAAa;MAC7B,eAAe,KAAK;MACpB,IAAI,KAAK,iBAAiB,cACxB,OAAO;KAEX;KACA,MAAM,YAAY,WAAW,KAAKM,KAAK;KACvC,IAAI,aAAa,GACf,MAAM,IAAI,mCACR,qCAAqC,aAAa,0BACzB,cAC3B;KAEF,MAAM,KAAKkB,wBACT,cACA,WACA,QAAQ,MACV;IACF;GACF,GAAG;GAEH,MAAM,kBAAkB,YAAY,QAAQ,WAAW;IACrD,KAAKJ,gBAAgB,MAAM;IAG3B,IAAI,CAAC,WAAW,aACd,OAAO,IAAI,cAAqB,CAAC,CAAC;IAEpC,KAAKV,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,UAAS,CAAC;IAIxD,KAAKD,yBAAyB,IAAI,CAAC;IACnC,MAAM,IAAI,mCACR,0BAA0B,WAAW,GAAG,6DACA,aAAa,0BAC5B,cAC3B;GACF,CAAC;GAED,OAAO,MAAM,QAAQ,KAAK,CAAC,WAAW,eAAe,CAAC;EACxD,UAAU;GACR,OAAO,oBAAoB,SAAS,YAAY;GAChD,QAAQ,MAAM;GACd,YAAY,OAAO;EACrB;CACF;;;;;;;;;CAUA,MAAMe,wBACJ,cACA,aACA,QACe;EACf,MAAM,YAAY,SAAe;EACjC,MAAM,SAAS;GAAC,WAAW;GAAc,SAAS,UAAU;EAAO;EACnE,KAAKN,eAAe,IAAI,MAAM;EAG9B,MAAM,OAAO,IAAI,gBAAgB;EACjC,MAAM,kBAAkB,KAAK,MAAM;EACnC,OAAO,iBAAiB,SAAS,WAAW,EAAC,MAAM,KAAI,CAAC;EACxD,IAAI;GACF,MAAM,UAAU,MAAM,QAAQ,KAAK,CACjC,UAAU,QAAQ,WAAW,QAAiB,GAC9C,KAAKb,OACH,KAAK,IAAI,KAAKF,wBAAwB,WAAW,GACjD,KAAK,MACP,EAAE,WAAW,MAAe,CAC9B,CAAC;GACD,KAAKO,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,QAAO,CAAC;EACxD,UAAU;GACR,KAAKQ,eAAe,OAAO,MAAM;GACjC,OAAO,oBAAoB,SAAS,SAAS;GAC7C,KAAK,MAAM;EACb;CACF;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAKC,WAAW,OAAO,SACzB,MAAM,IAAI,WAAW,mCAAmC;CAE5D;AACF;;;;;;AASA,IAAa,8BAAb,cAAiD,MAAM,CAAC;;AAGxD,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB;;AAGA,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.d.ts

@@ -0,0 +1,83 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { ShardID } from '../../types/shards.ts';
+import { type ChangeLogIdentity } from '../replicator/change-log-db.ts';
+import { type ChangeLogEntry } from './change-log-codec.ts';
+/**
+ * Result for one committed transaction.
+ *
+ * `mismatch` covers missing, extra, changed, or reordered rows.
+ * `inconclusive` means that store bounds changed during the comparison.
+ */
+export type CompareOutcome = 'match' | 'mismatch' | 'inconclusive' | 'deferred' | 'oversized';
+export type CompareSkipReason = 'stopped' | 'log-rebuilt' | 'log-unavailable' | 'ineligible-schema' | 'ineligible-identity' | 'cold-log' | 'nothing-to-compare' | 'error';
+export type CompareCycleResult = {
+    readonly kind: 'skipped';
+    readonly reason: CompareSkipReason;
+} | {
+    readonly kind: 'compared';
+    /** Exclusive lower bound of the compared range. */
+    readonly fromWatermark: string;
+    /** Inclusive upper bound handled. The next cycle resumes from here. */
+    readonly throughWatermark: string;
+    /** Committed transactions handled through `throughWatermark`. */
+    readonly transactions: number;
+    /** Transactions with compared catchup output. */
+    readonly sampled: number;
+    readonly matched: number;
+    readonly mismatched: number;
+    /** Suspects rejected because store bounds changed. */
+    readonly inconclusive: number;
+    /** Sampled transactions deferred to a fresh budget. */
+    readonly deferred: number;
+    /** Sampled transactions that exceed a fresh budget. */
+    readonly oversized: number;
+};
+/** Provides the Postgres side of the catchup comparison. */
+export interface PGChangeLogRangeReader {
+    getCatchupBounds(): Promise<{
+        minWatermark: string | null;
+        lastWatermark: string;
+    }>;
+    listCommitWatermarks(afterWatermark: string, throughWatermark: string, limit: number): Promise<string[]>;
+    readCatchupRange(afterWatermark: string, throughWatermark: string, batchRows?: number | undefined): AsyncIterable<ChangeLogEntry[]>;
+}
+export type SQLiteChangeLogCompareOptions = {
+    /** Stable percentage of transactions selected for payload comparison. */
+    readonly comparePercent: number;
+    /** Warm-up time after a seed. The comparator skips a younger log. */
+    readonly retentionMs: number;
+    readonly intervalMs?: number | undefined;
+    readonly maxTransactionsPerCycle?: number | undefined;
+    /** Maximum catchup rows read from each store in one cycle. */
+    readonly maxRowsPerSourcePerCycle?: number | undefined;
+    /** Maximum catchup bytes read from each store in one cycle. */
+    readonly maxBytesPerSourcePerCycle?: number | undefined;
+    readonly readBatchRows?: number | undefined;
+    /** Clock for the warm-up period. Defaults to `Date.now`. */
+    readonly now?: (() => number) | undefined;
+    readonly setTimeoutFn?: typeof setTimeout | undefined;
+    readonly clearTimeoutFn?: typeof clearTimeout | undefined;
+    /** Yields between sampled transactions. Defaults to `setImmediate`. */
+    readonly yieldFn?: (() => Promise<void>) | undefined;
+};
+/**
+ * Compares SQLite and Postgres catchup output without affecting subscribers.
+ *
+ * Each cycle compares complete ranges through the lower store head.
+ * It checks every transaction for presence and samples payloads.
+ * A bounds change makes the affected result `inconclusive`.
+ * Postgres remains authoritative.
+ */
+export declare class SQLiteChangeLogComparator {
+    #private;
+    constructor(lc: LogContext, shard: ShardID, changeLogFile: string, identity: ChangeLogIdentity, pg: PGChangeLogRangeReader, opts: SQLiteChangeLogCompareOptions);
+    /**
+     * Runs one cycle. Concurrent calls share the same promise.
+     * Errors produce `skipped: 'error'`.
+     */
+    compareOnce(): Promise<CompareCycleResult>;
+    stop(): void;
+    /** Invalidates cycles that can still read a replaced file. */
+    invalidate(): void;
+}
+//# sourceMappingURL=sqlite-change-log-comparator.d.ts.map

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite-change-log-comparator.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-comparator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAIL,KAAK,iBAAiB,EAEvB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AA8B/B;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GACtB,OAAO,GACP,UAAU,GACV,cAAc,GACd,UAAU,GACV,WAAW,CAAC;AAEhB,MAAM,MAAM,iBAAiB,GAEzB,SAAS,GAET,aAAa,GAEb,iBAAiB,GAEjB,mBAAmB,GAEnB,qBAAqB,GAErB,UAAU,GAEV,oBAAoB,GAEpB,OAAO,CAAC;AAEZ,MAAM,MAAM,kBAAkB,GAC1B;IAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAA;CAAC,GAC9D;IACE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,mDAAmD;IACnD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,uEAAuE;IACvE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,iEAAiE;IACjE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,iDAAiD;IACjD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,sDAAsD;IACtD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,uDAAuD;IACvD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,uDAAuD;IACvD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEN,4DAA4D;AAC5D,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,IAAI,OAAO,CAAC;QAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,oBAAoB,CAClB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACrB,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,GAC7B,aAAa,CAAC,cAAc,EAAE,CAAC,CAAC;CACpC;AAED,MAAM,MAAM,6BAA6B,GAAG;IAC1C,yEAAyE;IACzE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,qEAAqE;IACrE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtD,8DAA8D;IAC9D,QAAQ,CAAC,wBAAwB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvD,+DAA+D;IAC/D,QAAQ,CAAC,yBAAyB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxD,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5C,4DAA4D;IAC5D,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IACtD,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;IAC1D,uEAAuE;IACvE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC;CACtD,CAAC;AAyCF;;;;;;;GAOG;AACH,qBAAa,yBAAyB;;gBAuClC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,iBAAiB,EAC3B,EAAE,EAAE,sBAAsB,EAC1B,IAAI,EAAE,6BAA6B;IAiBrC;;;OAGG;IACH,WAAW,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAmB1C,IAAI,IAAI,IAAI;IAQZ,8DAA8D;IAC9D,UAAU,IAAI,IAAI;CAifnB"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.js

@@ -0,0 +1,417 @@
+import { Database } from "../../../../zqlite/src/db.js";
+import { getOrCreateCounter, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
+import { CHANGE_LOG_STREAM_TABLE, readChangeLogMeta } from "../replicator/change-log-db.js";
+import { reconstructWatermarkedChange } from "./change-log-codec.js";
+import { SQLiteChangeLogReader } from "./sqlite-change-log-reader.js";
+import { digestCatchupRange } from "./change-log-compare-digest.js";
+import { isSampledForShard } from "./shard-sampling.js";
+//#region ../zero-cache/src/services/change-streamer/sqlite-change-log-comparator.ts
+/** Interval between comparison cycles. */
+var DEFAULT_COMPARE_INTERVAL_MS = 3e4;
+/** Maximum transactions enumerated per cycle. */
+var DEFAULT_MAX_TRANSACTIONS_PER_CYCLE = 256;
+/** Maximum catchup rows read from each store per cycle. */
+var DEFAULT_MAX_ROWS_PER_SOURCE_PER_CYCLE = 1e3;
+/**
+* Maximum catchup bytes read from each store per cycle.
+*
+* Rows bound the per-row overhead. Bytes bound the read and hash work,
+* which scales with payload size instead of row count. Normal traffic
+* reaches the row limit first. Wide payloads reach this one.
+*/
+var DEFAULT_MAX_BYTES_PER_SOURCE_PER_CYCLE = 32 * 1024 * 1024;
+/** Rows per SQLite catchup batch. */
+var DEFAULT_READ_BATCH_ROWS = 1e3;
+/** Uses separate aggregates so SQLite can use an index for each value. */
+var SQLITE_BOUNDS_SQL = `
+  SELECT
+    (SELECT min("watermark") FROM "${CHANGE_LOG_STREAM_TABLE}")
+      AS "minWatermark",
+    (SELECT max("watermark") FROM "${CHANGE_LOG_STREAM_TABLE}")
+      AS "headWatermark"
+`;
+/** Lists committed transaction watermarks in `(@after, @through]`. */
+var SQLITE_COMMITS_SQL = `
+  SELECT "watermark" FROM "${CHANGE_LOG_STREAM_TABLE}"
+   WHERE "precommit" IS NOT NULL
+     AND "watermark" > @after
+     AND "watermark" <= @through
+   ORDER BY "watermark"
+   LIMIT @limit
+`;
+/**
+* Compares SQLite and Postgres catchup output without affecting subscribers.
+*
+* Each cycle compares complete ranges through the lower store head.
+* It checks every transaction for presence and samples payloads.
+* A bounds change makes the affected result `inconclusive`.
+* Postgres remains authoritative.
+*/
+var SQLiteChangeLogComparator = class {
+	#lc;
+	#shard;
+	#changeLogFile;
+	#identity;
+	#pg;
+	#opts;
+	#now;
+	#setTimeoutFn;
+	#clearTimeoutFn;
+	#yield;
+	#compareResults = getOrCreateCounter("replica", "sqlite_change_log.compare_result", "Outcomes of the dark comparison between what SQLite catchup serves and what PG catchup serves, per committed transaction. `inconclusive` means a purge or reseed moved the pinned bounds mid-comparison.");
+	#compareCycles = getOrCreateCounter("replica", "sqlite_change_log.compare_cycles", "SQLite change-log comparison cycles, by result.");
+	#cycleDuration = getOrCreateLatencyHistogram("replica", "sqlite_change_log.compare_cycle_duration", "Duration of one SQLite change-log comparison cycle.");
+	/** Last handled commit watermark. A restart derives it from current bounds. */
+	#cursor;
+	#running;
+	#timer;
+	#stopped = false;
+	#continueWithoutDelay = false;
+	#fileGeneration = 0;
+	constructor(lc, shard, changeLogFile, identity, pg, opts) {
+		this.#lc = lc.withContext("component", "sqlite-change-log-compare");
+		this.#shard = shard;
+		this.#changeLogFile = changeLogFile;
+		this.#identity = identity;
+		this.#pg = pg;
+		this.#opts = opts;
+		this.#now = opts.now ?? Date.now;
+		this.#setTimeoutFn = opts.setTimeoutFn ?? setTimeout;
+		this.#clearTimeoutFn = opts.clearTimeoutFn ?? clearTimeout;
+		this.#yield = opts.yieldFn ?? (() => new Promise((resolve) => setImmediate(resolve)));
+		this.#scheduleNext();
+	}
+	/**
+	* Runs one cycle. Concurrent calls share the same promise.
+	* Errors produce `skipped: 'error'`.
+	*/
+	compareOnce() {
+		if (this.#stopped) return Promise.resolve({
+			kind: "skipped",
+			reason: "stopped"
+		});
+		if (this.#running === void 0) this.#running = this.#runCycle().then((result) => {
+			if (this.#continueWithoutDelay) this.#rescheduleNext(0);
+			return result;
+		}).finally(() => {
+			this.#running = void 0;
+		});
+		return this.#running;
+	}
+	stop() {
+		this.#stopped = true;
+		if (this.#timer !== void 0) {
+			this.#clearTimeoutFn(this.#timer);
+			this.#timer = void 0;
+		}
+	}
+	/** Invalidates cycles that can still read a replaced file. */
+	invalidate() {
+		this.#fileGeneration++;
+		this.#cursor = void 0;
+		this.#continueWithoutDelay = false;
+	}
+	#scheduleNext(delayMs = this.#opts.intervalMs ?? DEFAULT_COMPARE_INTERVAL_MS) {
+		if (this.#stopped || this.#timer !== void 0) return;
+		this.#timer = this.#setTimeoutFn(() => {
+			this.#timer = void 0;
+			this.compareOnce().finally(() => this.#scheduleNext());
+		}, delayMs);
+	}
+	#rescheduleNext(delayMs) {
+		if (this.#timer !== void 0) {
+			this.#clearTimeoutFn(this.#timer);
+			this.#timer = void 0;
+		}
+		this.#scheduleNext(delayMs);
+	}
+	async #runCycle() {
+		const startedAt = performance.now();
+		const fileGeneration = this.#fileGeneration;
+		this.#continueWithoutDelay = false;
+		let db;
+		let reader;
+		try {
+			try {
+				db = new Database(this.#lc, this.#changeLogFile, { readonly: true });
+			} catch {
+				return this.#skipped("log-unavailable");
+			}
+			const pinned = await this.#pinBounds(db);
+			if (fileGeneration !== this.#fileGeneration) return this.#skipped("log-rebuilt");
+			if (typeof pinned === "string") return this.#skipped(pinned);
+			const { meta } = pinned;
+			const from = maxWatermark(this.#cursor ?? meta.seedWatermark, meta.seedWatermark, pinned.pgMinWatermark, pinned.sqlite.minWatermark);
+			const through = minWatermark(pinned.pgLastWatermark, pinned.sqlite.headWatermark);
+			if (through <= from) return this.#skipped("nothing-to-compare");
+			let transactions = 0;
+			let sampled = 0;
+			const outcomes = {
+				match: 0,
+				mismatch: 0,
+				inconclusive: 0,
+				deferred: 0,
+				oversized: 0
+			};
+			const record = (watermark, outcome) => {
+				this.#compareResults.add(1, { outcome });
+				outcomes[outcome]++;
+				if (outcome === "mismatch") this.#lc.error?.("SQLite change-log catchup output diverged from Postgres", { sqliteChangeLogCompare: { watermark } });
+			};
+			reader = new SQLiteChangeLogReader(this.#lc, this.#changeLogFile);
+			const plan = reader.plan(from);
+			if (plan.kind === "not-ready") return this.#skipped("log-unavailable");
+			if (plan.kind === "ahead") return this.#skipped("nothing-to-compare");
+			if (plan.kind === "too-old") {
+				const reconfirmed = await this.#reconfirm(db, pinned, from);
+				const outcome = fileGeneration === this.#fileGeneration ? reconfirmed : "inconclusive";
+				record(from, outcome);
+				if (outcome === "inconclusive") return this.#compared(from, from, transactions, sampled, outcomes);
+			}
+			const limit = Math.max(1, this.#opts.maxTransactionsPerCycle ?? DEFAULT_MAX_TRANSACTIONS_PER_CYCLE);
+			const union = await this.#enumerateCommits(db, from, through, limit);
+			if (fileGeneration !== this.#fileGeneration) return this.#skipped("log-rebuilt");
+			const maxRowsPerSource = Math.max(1, this.#opts.maxRowsPerSourcePerCycle ?? DEFAULT_MAX_ROWS_PER_SOURCE_PER_CYCLE);
+			const maxBytesPerSource = Math.max(1, this.#opts.maxBytesPerSourcePerCycle ?? DEFAULT_MAX_BYTES_PER_SOURCE_PER_CYCLE);
+			let sqliteRowsRead = 0;
+			let pgRowsRead = 0;
+			let sqliteBytesRead = 0;
+			let pgBytesRead = 0;
+			let previousBoundary = from;
+			let handledThrough = from;
+			for (const { watermark, inSqlite, inPg } of union) {
+				if (this.#stopped) break;
+				let stopAfterTransaction = false;
+				if (!inSqlite || !inPg) {
+					const outcome = await this.#reconfirm(db, pinned, watermark);
+					record(watermark, fileGeneration === this.#fileGeneration ? outcome : "inconclusive");
+				} else if (isSampledForShard(this.#shard, watermark, this.#opts.comparePercent)) {
+					const sqliteRowsRemaining = maxRowsPerSource - sqliteRowsRead;
+					const pgRowsRemaining = maxRowsPerSource - pgRowsRead;
+					const sqliteBytesRemaining = maxBytesPerSource - sqliteBytesRead;
+					const pgBytesRemaining = maxBytesPerSource - pgBytesRead;
+					if (sqliteRowsRemaining === 0 || pgRowsRemaining === 0 || sqliteBytesRemaining <= 0 || pgBytesRemaining <= 0) {
+						record(watermark, "deferred");
+						break;
+					}
+					sampled++;
+					const comparison = await this.#compareTransaction(db, reader, pinned, previousBoundary, watermark, sqliteRowsRemaining, pgRowsRemaining, sqliteBytesRemaining, pgBytesRemaining, maxRowsPerSource, maxBytesPerSource);
+					sqliteRowsRead += comparison.sqliteRowsRead;
+					pgRowsRead += comparison.pgRowsRead;
+					sqliteBytesRead += comparison.sqliteBytesRead;
+					pgBytesRead += comparison.pgBytesRead;
+					const outcome = fileGeneration === this.#fileGeneration ? comparison.outcome : "inconclusive";
+					record(watermark, outcome);
+					if (outcome === "inconclusive") break;
+					if (outcome === "deferred") break;
+					if (outcome === "oversized") stopAfterTransaction = true;
+					await this.#yield();
+				}
+				previousBoundary = watermark;
+				handledThrough = watermark;
+				transactions++;
+				if (stopAfterTransaction) break;
+			}
+			if (!this.#stopped && fileGeneration === this.#fileGeneration && outcomes.inconclusive === 0) {
+				this.#cursor = handledThrough;
+				this.#continueWithoutDelay = handledThrough < through;
+			}
+			return this.#compared(from, handledThrough, transactions, sampled, outcomes);
+		} catch (e) {
+			this.#lc.warn?.("error comparing the SQLite change log", e);
+			return this.#skipped("error");
+		} finally {
+			reader?.close();
+			db?.close();
+			this.#cycleDuration.recordMs(performance.now() - startedAt);
+		}
+	}
+	/** Checks log eligibility and reads the bounds from both stores. */
+	async #pinBounds(db) {
+		let meta;
+		try {
+			meta = readChangeLogMeta(db);
+		} catch {
+			return "log-unavailable";
+		}
+		if (meta.schemaVersion !== 4) return "ineligible-schema";
+		const { epoch, generation, replicaID } = this.#identity;
+		if (meta.epoch !== epoch || meta.generation !== generation || meta.replicaID !== replicaID) return "ineligible-identity";
+		if (this.#now() - meta.seededAtMs < this.#opts.retentionMs) return "cold-log";
+		const sqlite = this.#readSQLiteBounds(db);
+		if (sqlite === void 0) return "log-unavailable";
+		const { minWatermark, lastWatermark } = await this.#pg.getCatchupBounds();
+		if (minWatermark === null) return "nothing-to-compare";
+		return {
+			meta,
+			sqlite,
+			pgMinWatermark: minWatermark,
+			pgLastWatermark: lastWatermark
+		};
+	}
+	#readSQLiteBounds(db) {
+		const bounds = db.prepare(SQLITE_BOUNDS_SQL).get();
+		return bounds.minWatermark === null || bounds.headWatermark === null ? void 0 : {
+			minWatermark: bounds.minWatermark,
+			headWatermark: bounds.headWatermark
+		};
+	}
+	/**
+	* Merges committed transaction watermarks from both stores.
+	* If a list reaches the limit, the result ends at the last complete common range.
+	*/
+	async #enumerateCommits(db, from, through, limit) {
+		let sqlite = db.prepare(SQLITE_COMMITS_SQL).all({
+			after: from,
+			through,
+			limit: limit + 1
+		}).map(({ watermark }) => watermark);
+		let pg = await this.#pg.listCommitWatermarks(from, through, limit + 1);
+		let effectiveThrough = through;
+		if (sqlite.length > limit) {
+			sqlite.length = limit;
+			effectiveThrough = minWatermark(effectiveThrough, sqlite[limit - 1]);
+		}
+		if (pg.length > limit) {
+			pg.length = limit;
+			effectiveThrough = minWatermark(effectiveThrough, pg[limit - 1]);
+		}
+		if (effectiveThrough !== through) {
+			sqlite = sqlite.filter((w) => w <= effectiveThrough);
+			pg = pg.filter((w) => w <= effectiveThrough);
+		}
+		const union = [];
+		let s = 0;
+		let p = 0;
+		while (s < sqlite.length || p < pg.length) {
+			const sw = s < sqlite.length ? sqlite[s] : void 0;
+			const pw = p < pg.length ? pg[p] : void 0;
+			if (sw !== void 0 && (pw === void 0 || sw < pw)) {
+				union.push({
+					watermark: sw,
+					inSqlite: true,
+					inPg: false
+				});
+				s++;
+			} else if (pw !== void 0 && (sw === void 0 || pw < sw)) {
+				union.push({
+					watermark: pw,
+					inSqlite: false,
+					inPg: true
+				});
+				p++;
+			} else if (sw !== void 0) {
+				union.push({
+					watermark: sw,
+					inSqlite: true,
+					inPg: true
+				});
+				s++;
+				p++;
+			}
+		}
+		return union;
+	}
+	/**
+	* Compares the output in `(previousBoundary, watermark]`.
+	* Positional reads do not require either store to contain the boundary row.
+	*/
+	async #compareTransaction(db, reader, pinned, previousBoundary, watermark, sqliteMaxRows, pgMaxRows, sqliteMaxBytes, pgMaxBytes, maxRowsPerSource, maxBytesPerSource) {
+		let sqlite;
+		let pg;
+		try {
+			const readBatchRows = this.#opts.readBatchRows ?? DEFAULT_READ_BATCH_ROWS;
+			sqlite = await digestCatchupRange(reader.read(previousBoundary, watermark, Math.min(readBatchRows, sqliteMaxRows), void 0, sqliteMaxRows), watermark, sqliteMaxRows, sqliteMaxBytes);
+			pg = await digestCatchupRange(reconstructBatches(this.#pg.readCatchupRange(previousBoundary, watermark, pgMaxRows)), watermark, pgMaxRows, pgMaxBytes);
+		} catch (e) {
+			this.#lc.warn?.(`error reading transaction ${watermark} for comparison`, e);
+			return {
+				outcome: await this.#reconfirm(db, pinned, watermark),
+				sqliteRowsRead: 0,
+				pgRowsRead: 0,
+				sqliteBytesRead: 0,
+				pgBytesRead: 0
+			};
+		}
+		const read = {
+			sqliteRowsRead: sqlite.rows,
+			pgRowsRead: pg.rows,
+			sqliteBytesRead: sqlite.bytes,
+			pgBytesRead: pg.bytes
+		};
+		if (sqlite.limitReached || pg.limitReached) {
+			const sqliteFresh = sqliteMaxRows === maxRowsPerSource && sqliteMaxBytes === maxBytesPerSource;
+			const pgFresh = pgMaxRows === maxRowsPerSource && pgMaxBytes === maxBytesPerSource;
+			return {
+				outcome: sqlite.limitReached && sqliteFresh || pg.limitReached && pgFresh ? "oversized" : "deferred",
+				...read
+			};
+		}
+		if (sqlite.digest === pg.digest) return {
+			outcome: "match",
+			...read
+		};
+		return {
+			outcome: await this.#reconfirm(db, pinned, watermark),
+			...read
+		};
+	}
+	/**
+	* Rechecks a suspect result against current store bounds.
+	* A moved bound, changed seed, failed read, or lower head makes the result inconclusive.
+	*/
+	async #reconfirm(db, pinned, watermark) {
+		try {
+			const meta = readChangeLogMeta(db);
+			if (meta.seedWatermark !== pinned.meta.seedWatermark || meta.seededAtMs !== pinned.meta.seededAtMs) return "inconclusive";
+			const sqlite = this.#readSQLiteBounds(db);
+			const { minWatermark, lastWatermark } = await this.#pg.getCatchupBounds();
+			if (sqlite === void 0 || minWatermark === null) return "inconclusive";
+			if (sqlite.minWatermark > pinned.sqlite.minWatermark && watermark <= sqlite.minWatermark || minWatermark > pinned.pgMinWatermark && watermark <= minWatermark || watermark > sqlite.headWatermark || watermark > lastWatermark) return "inconclusive";
+			return "mismatch";
+		} catch {
+			return "inconclusive";
+		}
+	}
+	#skipped(reason) {
+		this.#compareCycles.add(1, { result: reason });
+		if (reason === "error") {} else if (reason !== "nothing-to-compare") this.#lc.debug?.(`skipping SQLite change-log comparison: ${reason}`);
+		return {
+			kind: "skipped",
+			reason
+		};
+	}
+	#compared(fromWatermark, throughWatermark, transactions, sampled, outcomes) {
+		this.#compareCycles.add(1, { result: "compared" });
+		const result = {
+			kind: "compared",
+			fromWatermark,
+			throughWatermark,
+			transactions,
+			sampled,
+			matched: outcomes.match,
+			mismatched: outcomes.mismatch,
+			inconclusive: outcomes.inconclusive,
+			deferred: outcomes.deferred,
+			oversized: outcomes.oversized
+		};
+		this.#lc.debug?.("SQLite change-log comparison cycle", { sqliteChangeLogCompare: result });
+		return result;
+	}
+};
+/** Applies the same reconstruction as Postgres catchup. */
+async function* reconstructBatches(batches) {
+	for await (const batch of batches) yield batch.map(reconstructWatermarkedChange);
+}
+function minWatermark(a, b) {
+	return a < b ? a : b;
+}
+function maxWatermark(...watermarks) {
+	let max = watermarks[0];
+	for (const w of watermarks) if (w > max) max = w;
+	return max;
+}
+//#endregion
+export { SQLiteChangeLogComparator };
+
+//# sourceMappingURL=sqlite-change-log-comparator.js.map