@rocicorp/zero 1.9.0-canary.1 → 1.9.0-canary.3

package/out/zero-cache/src/services/replicator/incremental-sync.js.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type Downstream,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport type {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(lc);\n  }\n\n  async run() {\n    const lc = this.#lc;\n    this.#worker.onError(err => this.#state.stop(lc, err));\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    // This initial notification intentionally omits replicaReadyTimeMs because\n    // it represents already-current state, not newly-unserved work.\n    void this.#notifier.notifySubscribers({\n      state: 'version-ready',\n      watermark: initialWatermark,\n    });\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Source<Downstream> | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        downstream = await this.#changeStreamer.subscribe({\n          protocolVersion: PROTOCOL_VERSION,\n          taskID: this.#taskID,\n          id: this.#id,\n          mode: this.#mode,\n          watermark,\n          replicaVersion,\n          initial: watermark === initialWatermark,\n        });\n        this.#state.resetBackoff();\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfillStatus: DownloadStatus | undefined;\n\n        for await (const message of downstream) {\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'status': {\n              const {lagReport} = message[1];\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              const {type, message: msg} = message[1];\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status} = msg;\n                if (!backfillStatus) {\n                  // Start publishing the status every 3 seconds.\n                  backfillStatus = status;\n                  this.#statusPublisher?.publish(\n                    lc,\n                    'Replicating',\n                    `Backfilling ${msg.relation.name} table`,\n                    3000,\n                    () =>\n                      backfillStatus\n                        ? {\n                            downloadStatus: [\n                              {\n                                ...backfillStatus,\n                                table: msg.relation.name,\n                                columns: [\n                                  ...msg.relation.rowKey.columns,\n                                  ...msg.columns,\n                                ],\n                              },\n                            ],\n                          }\n                        : {},\n                  );\n                }\n                backfillStatus = status; // Update the current status\n              }\n\n              const result = await this.#worker.processMessage(\n                message as ChangeStreamData,\n              );\n\n              this.#handleResult(lc, result);\n              if (result?.completedBackfill) {\n                backfillStatus = undefined;\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({\n        state: 'version-ready',\n        watermark: result.watermark,\n        replicaReadyTimeMs: Date.now(),\n      });\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAgCA,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BAA0B,EAAE;CACnD;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKR;EAChB,KAAKI,QAAQ,SAAQ,QAAO,KAAKK,OAAO,KAAK,IAAI,GAAG,CAAC;EACrD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKL,QAAQ,qBAAqB;EAK1C,KAAUG,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAKE,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,KAAKL,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,aAAa,MAAM,KAAKD,gBAAgB,UAAU;KAChD,iBAAA;KACA,QAAQ,KAAKF;KACb,IAAI,KAAKC;KACT,MAAM,KAAKG;KACX;KACA;KACA,SAAS,cAAc;IACzB,CAAC;IACD,KAAKI,OAAO,aAAa;IACzB,aAAa,KAAKA,OAAO,aAAa,UAAU;IAChD,KAAKH,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAEJ,WAAW,MAAM,WAAW,YAAY;KACtC,KAAKI,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;OAC5B,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKF,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;OACrC,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,WAAU;QACjB,IAAI,CAAC,gBAAgB;SAEnB,iBAAiB;SACjB,KAAKF,kBAAkB,QACrB,IACA,eACA,eAAe,IAAI,SAAS,KAAK,SACjC,WAEE,iBACI,EACE,gBAAgB,CACd;UACE,GAAG;UACH,OAAO,IAAI,SAAS;UACpB,SAAS,CACP,GAAG,IAAI,SAAS,OAAO,SACvB,GAAG,IAAI,OACT;SACF,CACF,EACF,IACA,CAAC,CACT;QACF;QACA,iBAAiB;OACnB;OAEA,MAAM,SAAS,MAAM,KAAKF,QAAQ,eAChC,OACF;OAEA,KAAKO,cAAc,IAAI,MAAM;OAC7B,IAAI,QAAQ,mBACV,iBAAiB,KAAA;OAEnB;MACF;KACF;IACF;IACA,KAAKP,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,MAAM,KAAKG,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;CACvC;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKH,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKE,OAAO,KAAK,IAAI,GAAG;CAC1B;AACF"}
+{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type Downstream,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport type {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(lc);\n  }\n\n  async run() {\n    const lc = this.#lc;\n    let workerError: Error | undefined;\n    this.#worker.onError(err => {\n      workerError ??= err;\n      this.#state.stop(lc, err);\n    });\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    // This initial notification intentionally omits replicaReadyTimeMs because\n    // it represents already-current state, not newly-unserved work.\n    void this.#notifier.notifySubscribers({\n      state: 'version-ready',\n      watermark: initialWatermark,\n    });\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Source<Downstream> | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        downstream = await this.#changeStreamer.subscribe({\n          protocolVersion: PROTOCOL_VERSION,\n          taskID: this.#taskID,\n          id: this.#id,\n          mode: this.#mode,\n          watermark,\n          replicaVersion,\n          initial: watermark === initialWatermark,\n        });\n        this.#state.resetBackoff();\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfillStatus: DownloadStatus | undefined;\n\n        for await (const message of downstream) {\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'status': {\n              const {lagReport} = message[1];\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              const {type, message: msg} = message[1];\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status} = msg;\n                if (!backfillStatus) {\n                  // Start publishing the status every 3 seconds.\n                  backfillStatus = status;\n                  this.#statusPublisher?.publish(\n                    lc,\n                    'Replicating',\n                    `Backfilling ${msg.relation.name} table`,\n                    3000,\n                    () =>\n                      backfillStatus\n                        ? {\n                            downloadStatus: [\n                              {\n                                ...backfillStatus,\n                                table: msg.relation.name,\n                                columns: [\n                                  ...msg.relation.rowKey.columns,\n                                  ...msg.columns,\n                                ],\n                              },\n                            ],\n                          }\n                        : {},\n                  );\n                }\n                backfillStatus = status; // Update the current status\n              }\n\n              const result = await this.#worker.processMessage(\n                message as ChangeStreamData,\n              );\n\n              this.#handleResult(lc, result);\n              if (result?.completedBackfill) {\n                backfillStatus = undefined;\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n    if (workerError) {\n      throw workerError;\n    }\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({\n        state: 'version-ready',\n        watermark: result.watermark,\n        replicaReadyTimeMs: Date.now(),\n        upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n      });\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAgCA,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BAA0B,EAAE;CACnD;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKR;EAChB,IAAI;EACJ,KAAKI,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAKK,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKL,QAAQ,qBAAqB;EAK1C,KAAUG,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAKE,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,KAAKL,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,aAAa,MAAM,KAAKD,gBAAgB,UAAU;KAChD,iBAAA;KACA,QAAQ,KAAKF;KACb,IAAI,KAAKC;KACT,MAAM,KAAKG;KACX;KACA;KACA,SAAS,cAAc;IACzB,CAAC;IACD,KAAKI,OAAO,aAAa;IACzB,aAAa,KAAKA,OAAO,aAAa,UAAU;IAChD,KAAKH,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAEJ,WAAW,MAAM,WAAW,YAAY;KACtC,KAAKI,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;OAC5B,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKF,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;OACrC,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,WAAU;QACjB,IAAI,CAAC,gBAAgB;SAEnB,iBAAiB;SACjB,KAAKF,kBAAkB,QACrB,IACA,eACA,eAAe,IAAI,SAAS,KAAK,SACjC,WAEE,iBACI,EACE,gBAAgB,CACd;UACE,GAAG;UACH,OAAO,IAAI,SAAS;UACpB,SAAS,CACP,GAAG,IAAI,SAAS,OAAO,SACvB,GAAG,IAAI,OACT;SACF,CACF,EACF,IACA,CAAC,CACT;QACF;QACA,iBAAiB;OACnB;OAEA,MAAM,SAAS,MAAM,KAAKF,QAAQ,eAChC,OACF;OAEA,KAAKO,cAAc,IAAI,MAAM;OAC7B,IAAI,QAAQ,mBACV,iBAAiB,KAAA;OAEnB;MACF;KACF;IACF;IACA,KAAKP,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,MAAM,KAAKG,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKH,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKE,OAAO,KAAK,IAAI,GAAG;CAC1B;AACF"}

package/out/zero-cache/src/services/replicator/notifier.d.ts

@@ -1,27 +1,5 @@
 import { type Result, Subscription } from '../../types/subscription.ts';
 import type { ReplicaState, ReplicaStateNotifier } from './replicator.ts';
-/**
- * Handles the semantics of {@link ReplicatorVersionNotifier.subscribe()}
- * notifications, namely:
- *
- * * New subscribers are notified immediately with the latest received
- *   ReplicaState.
- *
- * * Non-latest notifications are discarded if the subscriber is too
- *   busy to consume them.
- *
- * By encapsulating the state for the first behavior (essentially, whether
- * the first notification has been sent by the Replicator), Notifier objects
- * can be chained to simplify fanout from Replicator to View Syncers.
- *
- * In particular, each Syncer Thread can manage a single Subscription to
- * the Replicator across a MessageChannel, which it uses for its own Notifier
- * instance to manage subscriptions from View Syncers within its thread. This
- * way a Replicator only deals with sending notifications to a bounded set
- * of MessageChannel-based subscribers (Syncer Threads), while the dynamic
- * subscribe and unsubscribe traffic from View Syncers remains within each
- * Syncer Thread.
- */
 export declare class Notifier implements ReplicaStateNotifier {
     #private;
     get latestState(): ReplicaState | undefined;

package/out/zero-cache/src/services/replicator/notifier.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"notifier.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/notifier.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,MAAM,EACX,YAAY,EACb,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,YAAY,EAAE,oBAAoB,EAAC,MAAM,iBAAiB,CAAC;AAExE;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,QAAS,YAAW,oBAAoB;;IAInD,IAAI,WAAW,IAAI,YAAY,GAAG,SAAS,CAE1C;IAmBD,SAAS,IAAI,YAAY,CAAC,YAAY,CAAC;IAWvC,iBAAiB,CACf,KAAK,GAAE,YAAuC,GAC7C,OAAO,CAAC,MAAM,CAAC,EAAE;CAOrB"}
+{"version":3,"file":"notifier.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/notifier.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,MAAM,EACX,YAAY,EACb,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,YAAY,EAAE,oBAAoB,EAAC,MAAM,iBAAiB,CAAC;AAqCxE,qBAAa,QAAS,YAAW,oBAAoB;;IAInD,IAAI,WAAW,IAAI,YAAY,GAAG,SAAS,CAE1C;IAyBD,SAAS,IAAI,YAAY,CAAC,YAAY,CAAC;IAWvC,iBAAiB,CACf,KAAK,GAAE,YAAuC,GAC7C,OAAO,CAAC,MAAM,CAAC,EAAE;CAOrB"}

package/out/zero-cache/src/services/replicator/notifier.js

@@ -23,6 +23,11 @@ import { EventEmitter } from "eventemitter3";
 * subscribe and unsubscribe traffic from View Syncers remains within each
 * Syncer Thread.
 */
+function oldest(curr, prev) {
+	if (curr === void 0) return prev;
+	if (prev === void 0) return curr;
+	return Math.min(curr, prev);
+}
 var Notifier = class {
 	#eventEmitter = new EventEmitter();
 	#lastStateReceived;
@@ -34,7 +39,8 @@ var Notifier = class {
 		const subscription = Subscription.create({
 			coalesce: (curr, prev) => ({
 				...curr,
-				replicaReadyTimeMs: curr.replicaReadyTimeMs === void 0 ? prev.replicaReadyTimeMs : prev.replicaReadyTimeMs === void 0 ? curr.replicaReadyTimeMs : Math.min(curr.replicaReadyTimeMs, prev.replicaReadyTimeMs)
+				replicaReadyTimeMs: oldest(curr.replicaReadyTimeMs, prev.replicaReadyTimeMs),
+				upstreamCommitTimeMs: oldest(curr.upstreamCommitTimeMs, prev.upstreamCommitTimeMs)
 			}),
 			cleanup: () => this.#eventEmitter.off("version", notify)
 		});

package/out/zero-cache/src/services/replicator/notifier.js.map

@@ -1 +1 @@
-{"version":3,"file":"notifier.js","names":["#eventEmitter","#lastStateReceived","#newSubscription"],"sources":["../../../../../../zero-cache/src/services/replicator/notifier.ts"],"sourcesContent":["import {EventEmitter} from 'eventemitter3';\nimport {\n  type PendingResult,\n  type Result,\n  Subscription,\n} from '../../types/subscription.ts';\nimport type {ReplicaState, ReplicaStateNotifier} from './replicator.ts';\n\n/**\n * Handles the semantics of {@link ReplicatorVersionNotifier.subscribe()}\n * notifications, namely:\n *\n * * New subscribers are notified immediately with the latest received\n *   ReplicaState.\n *\n * * Non-latest notifications are discarded if the subscriber is too\n *   busy to consume them.\n *\n * By encapsulating the state for the first behavior (essentially, whether\n * the first notification has been sent by the Replicator), Notifier objects\n * can be chained to simplify fanout from Replicator to View Syncers.\n *\n * In particular, each Syncer Thread can manage a single Subscription to\n * the Replicator across a MessageChannel, which it uses for its own Notifier\n * instance to manage subscriptions from View Syncers within its thread. This\n * way a Replicator only deals with sending notifications to a bounded set\n * of MessageChannel-based subscribers (Syncer Threads), while the dynamic\n * subscribe and unsubscribe traffic from View Syncers remains within each\n * Syncer Thread.\n */\nexport class Notifier implements ReplicaStateNotifier {\n  readonly #eventEmitter = new EventEmitter();\n  #lastStateReceived: ReplicaState | undefined;\n\n  get latestState(): ReplicaState | undefined {\n    return this.#lastStateReceived;\n  }\n\n  #newSubscription() {\n    const notify = (state: ReplicaState) => subscription.push(state);\n    const subscription = Subscription.create<ReplicaState>({\n      coalesce: (curr, prev) => ({\n        ...curr,\n        replicaReadyTimeMs:\n          curr.replicaReadyTimeMs === undefined\n            ? prev.replicaReadyTimeMs\n            : prev.replicaReadyTimeMs === undefined\n              ? curr.replicaReadyTimeMs\n              : Math.min(curr.replicaReadyTimeMs, prev.replicaReadyTimeMs),\n      }),\n      cleanup: () => this.#eventEmitter.off('version', notify),\n    });\n    return {notify, subscription};\n  }\n\n  subscribe(): Subscription<ReplicaState> {\n    const {notify, subscription} = this.#newSubscription();\n    this.#eventEmitter.on('version', notify);\n    if (this.#lastStateReceived) {\n      // Per Replicator.subscribe() semantics, the current state of the\n      // replica, if known, is immediately sent on subscribe.\n      notify(this.#lastStateReceived);\n    }\n    return subscription;\n  }\n\n  notifySubscribers(\n    state: ReplicaState = {state: 'version-ready'},\n  ): Promise<Result>[] {\n    this.#lastStateReceived = state;\n    return this.#eventEmitter\n      .listeners('version')\n      .map(notify => notify(state) as unknown as PendingResult)\n      .map(pending => pending.result);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,IAAa,WAAb,MAAsD;CACpD,gBAAyB,IAAI,aAAa;CAC1C;CAEA,IAAI,cAAwC;EAC1C,OAAO,KAAKC;CACd;CAEA,mBAAmB;EACjB,MAAM,UAAU,UAAwB,aAAa,KAAK,KAAK;EAC/D,MAAM,eAAe,aAAa,OAAqB;GACrD,WAAW,MAAM,UAAU;IACzB,GAAG;IACH,oBACE,KAAK,uBAAuB,KAAA,IACxB,KAAK,qBACL,KAAK,uBAAuB,KAAA,IAC1B,KAAK,qBACL,KAAK,IAAI,KAAK,oBAAoB,KAAK,kBAAkB;GACnE;GACA,eAAe,KAAKD,cAAc,IAAI,WAAW,MAAM;EACzD,CAAC;EACD,OAAO;GAAC;GAAQ;EAAY;CAC9B;CAEA,YAAwC;EACtC,MAAM,EAAC,QAAQ,iBAAgB,KAAKE,iBAAiB;EACrD,KAAKF,cAAc,GAAG,WAAW,MAAM;EACvC,IAAI,KAAKC,oBAGP,OAAO,KAAKA,kBAAkB;EAEhC,OAAO;CACT;CAEA,kBACE,QAAsB,EAAC,OAAO,gBAAe,GAC1B;EACnB,KAAKA,qBAAqB;EAC1B,OAAO,KAAKD,cACT,UAAU,SAAS,EACnB,KAAI,WAAU,OAAO,KAAK,CAA6B,EACvD,KAAI,YAAW,QAAQ,MAAM;CAClC;AACF"}
+{"version":3,"file":"notifier.js","names":["#eventEmitter","#lastStateReceived","#newSubscription"],"sources":["../../../../../../zero-cache/src/services/replicator/notifier.ts"],"sourcesContent":["import {EventEmitter} from 'eventemitter3';\nimport {\n  type PendingResult,\n  type Result,\n  Subscription,\n} from '../../types/subscription.ts';\nimport type {ReplicaState, ReplicaStateNotifier} from './replicator.ts';\n\n/**\n * Handles the semantics of {@link ReplicatorVersionNotifier.subscribe()}\n * notifications, namely:\n *\n * * New subscribers are notified immediately with the latest received\n *   ReplicaState.\n *\n * * Non-latest notifications are discarded if the subscriber is too\n *   busy to consume them.\n *\n * By encapsulating the state for the first behavior (essentially, whether\n * the first notification has been sent by the Replicator), Notifier objects\n * can be chained to simplify fanout from Replicator to View Syncers.\n *\n * In particular, each Syncer Thread can manage a single Subscription to\n * the Replicator across a MessageChannel, which it uses for its own Notifier\n * instance to manage subscriptions from View Syncers within its thread. This\n * way a Replicator only deals with sending notifications to a bounded set\n * of MessageChannel-based subscribers (Syncer Threads), while the dynamic\n * subscribe and unsubscribe traffic from View Syncers remains within each\n * Syncer Thread.\n */\nfunction oldest(\n  curr: number | undefined,\n  prev: number | undefined,\n): number | undefined {\n  if (curr === undefined) {\n    return prev;\n  }\n  if (prev === undefined) {\n    return curr;\n  }\n  return Math.min(curr, prev);\n}\n\nexport class Notifier implements ReplicaStateNotifier {\n  readonly #eventEmitter = new EventEmitter();\n  #lastStateReceived: ReplicaState | undefined;\n\n  get latestState(): ReplicaState | undefined {\n    return this.#lastStateReceived;\n  }\n\n  #newSubscription() {\n    const notify = (state: ReplicaState) => subscription.push(state);\n    const subscription = Subscription.create<ReplicaState>({\n      // A coalesced notification stands in for every notification it\n      // subsumed, so both timestamps keep the *oldest* value: they measure how\n      // long work has been outstanding, and the subsumed watermarks are still\n      // unserved.\n      coalesce: (curr, prev) => ({\n        ...curr,\n        replicaReadyTimeMs: oldest(\n          curr.replicaReadyTimeMs,\n          prev.replicaReadyTimeMs,\n        ),\n        upstreamCommitTimeMs: oldest(\n          curr.upstreamCommitTimeMs,\n          prev.upstreamCommitTimeMs,\n        ),\n      }),\n      cleanup: () => this.#eventEmitter.off('version', notify),\n    });\n    return {notify, subscription};\n  }\n\n  subscribe(): Subscription<ReplicaState> {\n    const {notify, subscription} = this.#newSubscription();\n    this.#eventEmitter.on('version', notify);\n    if (this.#lastStateReceived) {\n      // Per Replicator.subscribe() semantics, the current state of the\n      // replica, if known, is immediately sent on subscribe.\n      notify(this.#lastStateReceived);\n    }\n    return subscription;\n  }\n\n  notifySubscribers(\n    state: ReplicaState = {state: 'version-ready'},\n  ): Promise<Result>[] {\n    this.#lastStateReceived = state;\n    return this.#eventEmitter\n      .listeners('version')\n      .map(notify => notify(state) as unknown as PendingResult)\n      .map(pending => pending.result);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAS,OACP,MACA,MACoB;CACpB,IAAI,SAAS,KAAA,GACX,OAAO;CAET,IAAI,SAAS,KAAA,GACX,OAAO;CAET,OAAO,KAAK,IAAI,MAAM,IAAI;AAC5B;AAEA,IAAa,WAAb,MAAsD;CACpD,gBAAyB,IAAI,aAAa;CAC1C;CAEA,IAAI,cAAwC;EAC1C,OAAO,KAAKC;CACd;CAEA,mBAAmB;EACjB,MAAM,UAAU,UAAwB,aAAa,KAAK,KAAK;EAC/D,MAAM,eAAe,aAAa,OAAqB;GAKrD,WAAW,MAAM,UAAU;IACzB,GAAG;IACH,oBAAoB,OAClB,KAAK,oBACL,KAAK,kBACP;IACA,sBAAsB,OACpB,KAAK,sBACL,KAAK,oBACP;GACF;GACA,eAAe,KAAKD,cAAc,IAAI,WAAW,MAAM;EACzD,CAAC;EACD,OAAO;GAAC;GAAQ;EAAY;CAC9B;CAEA,YAAwC;EACtC,MAAM,EAAC,QAAQ,iBAAgB,KAAKE,iBAAiB;EACrD,KAAKF,cAAc,GAAG,WAAW,MAAM;EACvC,IAAI,KAAKC,oBAGP,OAAO,KAAKA,kBAAkB;EAEhC,OAAO;CACT;CAEA,kBACE,QAAsB,EAAC,OAAO,gBAAe,GAC1B;EACnB,KAAKA,qBAAqB;EAC1B,OAAO,KAAKD,cACT,UAAU,SAAS,EACnB,KAAI,WAAU,OAAO,KAAK,CAA6B,EACvD,KAAI,YAAW,QAAQ,MAAM;CAClC;AACF"}

package/out/zero-cache/src/services/replicator/replicator.d.ts

@@ -3,7 +3,7 @@ import type { ReadonlyJSONObject } from '../../../../shared/src/json.ts';
 import type { Source } from '../../types/streams.ts';
 import type { ChangeStreamer } from '../change-streamer/change-streamer.ts';
 import type { Service } from '../service.ts';
-import type { ReplicationStatusPublisher } from './replication-status.ts';
+import { type ReplicationStatusPublisher } from './replication-status.ts';
 import type { WriteWorkerClient } from './write-worker-client.ts';
 /** See {@link ReplicaStateNotifier.subscribe()}. */
 export type ReplicaState = {
@@ -18,6 +18,21 @@ export type ReplicaState = {
      * Millisecond epoch when `watermark` became ready to read from the replica.
      */
     readonly replicaReadyTimeMs?: number | undefined;
+    /**
+     * Millisecond epoch at which the transaction behind `watermark` committed
+     * *upstream*, as reported by the ChangeSource. Unlike `replicaReadyTimeMs`,
+     * which is stamped locally when the change lands in the replica, this is the
+     * origin of the whole pipeline, so `now - upstreamCommitTimeMs` at the point
+     * the change is poked to clients is the end-to-end serving lag.
+     *
+     * Note that this crosses a clock domain: it comes from the upstream
+     * database, whereas it is subtracted from the local clock in the ViewSyncer.
+     * The existing `zero.replication.total_lag` metric spans the same boundary.
+     *
+     * Undefined when the ChangeSource does not report a commit time, and for the
+     * initial "current state is ready" notification.
+     */
+    readonly upstreamCommitTimeMs?: number | undefined;
     readonly testSeqNum?: number | undefined;
 };
 export interface ReplicaStateNotifier {

package/out/zero-cache/src/services/replicator/replicator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gCAAgC,CAAC;AACvE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAC1E,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C,OAAO,KAAK,EAAC,0BAA0B,EAAC,MAAM,yBAAyB,CAAC;AACxE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE,oDAAoD;AACpD,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAIjD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C,CAAC;AAEF,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,UAAW,SAAQ,oBAAoB;IACtD;;;;OAIG;IACH,MAAM,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACvC;AAED,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,CAAC;AAElD,qBAAa,iBAAkB,YAAW,UAAU,EAAE,OAAO;;IAC3D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAOlB,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,cAAc,EACpB,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,eAAe,EAAE,0BAA0B,GAAG,IAAI;IAmBpD,MAAM;;;IAIN,GAAG;IAKH,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAI3B,IAAI;CAQX"}
+{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gCAAgC,CAAC;AACvE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAC1E,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C,OAAO,EAEL,KAAK,0BAA0B,EAChC,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE,oDAAoD;AACpD,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEjD;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAInD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C,CAAC;AAEF,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,UAAW,SAAQ,oBAAoB;IACtD;;;;OAIG;IACH,MAAM,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACvC;AAED,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,CAAC;AAElD,qBAAa,iBAAkB,YAAW,UAAU,EAAE,OAAO;;IAC3D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAQlB,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,cAAc,EACpB,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,eAAe,EAAE,0BAA0B,GAAG,IAAI;IAoBpD,MAAM;;;IAIN,GAAG;IAcH,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAI3B,IAAI;CAQX"}

package/out/zero-cache/src/services/replicator/replicator.js

@@ -1,22 +1,28 @@
+import { publishReplicationError } from "./replication-status.js";
 import { IncrementalSyncer } from "./incremental-sync.js";
 //#region ../zero-cache/src/services/replicator/replicator.ts
 var ReplicatorService = class {
 	id;
 	#lc;
 	#incrementalSyncer;
+	#shouldPublishErrors;
 	#worker;
 	#runPromise;
 	constructor(lc, taskID, id, mode, changeStreamer, worker, statusPublisher) {
 		this.id = id;
 		this.#lc = lc.withContext("component", "replicator").withContext("serviceID", this.id);
 		this.#worker = worker;
+		this.#shouldPublishErrors = statusPublisher !== null;
 		this.#incrementalSyncer = new IncrementalSyncer(this.#lc, taskID, `${taskID}/${id}`, changeStreamer, worker, mode, statusPublisher);
 	}
 	status() {
 		return Promise.resolve({ status: "ok" });
 	}
 	run() {
-		this.#runPromise = this.#incrementalSyncer.run();
+		this.#runPromise = this.#incrementalSyncer.run().catch(async (error) => {
+			if (this.#shouldPublishErrors) await publishReplicationError(this.#lc, "Replicating", "Replication stopped because the replica writer failed");
+			throw error;
+		});
 		return this.#runPromise;
 	}
 	subscribe() {

package/out/zero-cache/src/services/replicator/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":["#lc","#incrementalSyncer","#worker","#runPromise"],"sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {ReadonlyJSONObject} from '../../../../shared/src/json.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {ChangeStreamer} from '../change-streamer/change-streamer.ts';\nimport type {Service} from '../service.ts';\nimport {IncrementalSyncer} from './incremental-sync.ts';\nimport type {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/** See {@link ReplicaStateNotifier.subscribe()}. */\nexport type ReplicaState = {\n  readonly state: 'version-ready';\n  /**\n   * The replica watermark that became ready. Omitted for the initial\n   * \"current state is ready\" notification, which should not count as newly\n   * unserved work for a freshly-started ViewSyncer.\n   */\n  readonly watermark?: string | undefined;\n  /**\n   * Millisecond epoch when `watermark` became ready to read from the replica.\n   */\n  readonly replicaReadyTimeMs?: number | undefined;\n\n  // Used in tests to verify behavior when additional information\n  // is ferried in the future. Not set in production.\n  readonly testSeqNum?: number | undefined;\n};\n\nexport interface ReplicaStateNotifier {\n  /**\n   * Creates a cancelable subscription of changes in the replica state.\n   *\n   * A `version-ready` message indicates that the replica is ready to be\n   * read, and henceforth that a _new_ version is ready, i.e. whenever a\n   * change is committed to the replica. The `version-ready` message itself\n   * otherwise contains no other information; the subscriber queries the\n   * replica for the current data.\n   *\n   * A `maintenance` state indicates that the replica should not be read from.\n   * If a subscriber is holding any transaction locks, it should release them\n   * until the next `version-ready` signal.\n   *\n   * Upon subscription, the current state of the replica is sent immediately\n   * if known. If multiple notifications occur before the subscriber\n   * can consume them, all but the last notification are discarded by the\n   * Subscription object (i.e. not buffered). Thus, a subscriber only\n   * ever consumes the current (i.e. known) state of the replica. This avoids\n   * a buildup of \"work\" if a subscriber is too busy to consume all\n   * notifications.\n   */\n  subscribe(): Source<ReplicaState>;\n}\n\nexport interface Replicator extends ReplicaStateNotifier {\n  /**\n   * Returns an opaque message for human-readable consumption. This is\n   * purely for ensuring that the Replicator has started at least once to\n   * bootstrap a new replica.\n   */\n  status(): Promise<ReadonlyJSONObject>;\n}\n\nexport type ReplicatorMode = 'backup' | 'serving';\n\nexport class ReplicatorService implements Replicator, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #incrementalSyncer: IncrementalSyncer;\n  readonly #worker: WriteWorkerClient;\n  #runPromise: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    mode: ReplicatorMode,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    statusPublisher: ReplicationStatusPublisher | null,\n  ) {\n    this.id = id;\n    this.#lc = lc\n      .withContext('component', 'replicator')\n      .withContext('serviceID', this.id);\n    this.#worker = worker;\n\n    this.#incrementalSyncer = new IncrementalSyncer(\n      this.#lc,\n      taskID,\n      `${taskID}/${id}`,\n      changeStreamer,\n      worker,\n      mode,\n      statusPublisher,\n    );\n  }\n\n  status() {\n    return Promise.resolve({status: 'ok'});\n  }\n\n  run() {\n    this.#runPromise = this.#incrementalSyncer.run();\n    return this.#runPromise;\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#incrementalSyncer.subscribe();\n  }\n\n  async stop() {\n    this.#incrementalSyncer.stop(this.#lc);\n    // Wait for the syncer's run loop to finish so that any in-flight\n    // worker.processMessage() call completes and clears #pending\n    // before we send the 'stop' message to the worker.\n    await this.#runPromise;\n    await this.#worker.stop();\n  }\n}\n"],"mappings":";;AAgEA,IAAa,oBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,IACA,MACA,gBACA,QACA,iBACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GACR,YAAY,aAAa,YAAY,EACrC,YAAY,aAAa,KAAK,EAAE;EACnC,KAAKE,UAAU;EAEf,KAAKD,qBAAqB,IAAI,kBAC5B,KAAKD,KACL,QACA,GAAG,OAAO,GAAG,MACb,gBACA,QACA,MACA,eACF;CACF;CAEA,SAAS;EACP,OAAO,QAAQ,QAAQ,EAAC,QAAQ,KAAI,CAAC;CACvC;CAEA,MAAM;EACJ,KAAKG,cAAc,KAAKF,mBAAmB,IAAI;EAC/C,OAAO,KAAKE;CACd;CAEA,YAAkC;EAChC,OAAO,KAAKF,mBAAmB,UAAU;CAC3C;CAEA,MAAM,OAAO;EACX,KAAKA,mBAAmB,KAAK,KAAKD,GAAG;EAIrC,MAAM,KAAKG;EACX,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF"}
+{"version":3,"file":"replicator.js","names":["#lc","#incrementalSyncer","#shouldPublishErrors","#worker","#runPromise"],"sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {ReadonlyJSONObject} from '../../../../shared/src/json.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {ChangeStreamer} from '../change-streamer/change-streamer.ts';\nimport type {Service} from '../service.ts';\nimport {IncrementalSyncer} from './incremental-sync.ts';\nimport {\n  publishReplicationError,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/** See {@link ReplicaStateNotifier.subscribe()}. */\nexport type ReplicaState = {\n  readonly state: 'version-ready';\n  /**\n   * The replica watermark that became ready. Omitted for the initial\n   * \"current state is ready\" notification, which should not count as newly\n   * unserved work for a freshly-started ViewSyncer.\n   */\n  readonly watermark?: string | undefined;\n  /**\n   * Millisecond epoch when `watermark` became ready to read from the replica.\n   */\n  readonly replicaReadyTimeMs?: number | undefined;\n\n  /**\n   * Millisecond epoch at which the transaction behind `watermark` committed\n   * *upstream*, as reported by the ChangeSource. Unlike `replicaReadyTimeMs`,\n   * which is stamped locally when the change lands in the replica, this is the\n   * origin of the whole pipeline, so `now - upstreamCommitTimeMs` at the point\n   * the change is poked to clients is the end-to-end serving lag.\n   *\n   * Note that this crosses a clock domain: it comes from the upstream\n   * database, whereas it is subtracted from the local clock in the ViewSyncer.\n   * The existing `zero.replication.total_lag` metric spans the same boundary.\n   *\n   * Undefined when the ChangeSource does not report a commit time, and for the\n   * initial \"current state is ready\" notification.\n   */\n  readonly upstreamCommitTimeMs?: number | undefined;\n\n  // Used in tests to verify behavior when additional information\n  // is ferried in the future. Not set in production.\n  readonly testSeqNum?: number | undefined;\n};\n\nexport interface ReplicaStateNotifier {\n  /**\n   * Creates a cancelable subscription of changes in the replica state.\n   *\n   * A `version-ready` message indicates that the replica is ready to be\n   * read, and henceforth that a _new_ version is ready, i.e. whenever a\n   * change is committed to the replica. The `version-ready` message itself\n   * otherwise contains no other information; the subscriber queries the\n   * replica for the current data.\n   *\n   * A `maintenance` state indicates that the replica should not be read from.\n   * If a subscriber is holding any transaction locks, it should release them\n   * until the next `version-ready` signal.\n   *\n   * Upon subscription, the current state of the replica is sent immediately\n   * if known. If multiple notifications occur before the subscriber\n   * can consume them, all but the last notification are discarded by the\n   * Subscription object (i.e. not buffered). Thus, a subscriber only\n   * ever consumes the current (i.e. known) state of the replica. This avoids\n   * a buildup of \"work\" if a subscriber is too busy to consume all\n   * notifications.\n   */\n  subscribe(): Source<ReplicaState>;\n}\n\nexport interface Replicator extends ReplicaStateNotifier {\n  /**\n   * Returns an opaque message for human-readable consumption. This is\n   * purely for ensuring that the Replicator has started at least once to\n   * bootstrap a new replica.\n   */\n  status(): Promise<ReadonlyJSONObject>;\n}\n\nexport type ReplicatorMode = 'backup' | 'serving';\n\nexport class ReplicatorService implements Replicator, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #incrementalSyncer: IncrementalSyncer;\n  readonly #shouldPublishErrors: boolean;\n  readonly #worker: WriteWorkerClient;\n  #runPromise: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    mode: ReplicatorMode,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    statusPublisher: ReplicationStatusPublisher | null,\n  ) {\n    this.id = id;\n    this.#lc = lc\n      .withContext('component', 'replicator')\n      .withContext('serviceID', this.id);\n    this.#worker = worker;\n    this.#shouldPublishErrors = statusPublisher !== null;\n\n    this.#incrementalSyncer = new IncrementalSyncer(\n      this.#lc,\n      taskID,\n      `${taskID}/${id}`,\n      changeStreamer,\n      worker,\n      mode,\n      statusPublisher,\n    );\n  }\n\n  status() {\n    return Promise.resolve({status: 'ok'});\n  }\n\n  run() {\n    this.#runPromise = this.#incrementalSyncer.run().catch(async error => {\n      if (this.#shouldPublishErrors) {\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          'Replication stopped because the replica writer failed',\n        );\n      }\n      throw error;\n    });\n    return this.#runPromise;\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#incrementalSyncer.subscribe();\n  }\n\n  async stop() {\n    this.#incrementalSyncer.stop(this.#lc);\n    // Wait for the syncer's run loop to finish so that any in-flight\n    // worker.processMessage() call completes and clears #pending\n    // before we send the 'stop' message to the worker.\n    await this.#runPromise;\n    await this.#worker.stop();\n  }\n}\n"],"mappings":";;;AAmFA,IAAa,oBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,IACA,MACA,gBACA,QACA,iBACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GACR,YAAY,aAAa,YAAY,EACrC,YAAY,aAAa,KAAK,EAAE;EACnC,KAAKG,UAAU;EACf,KAAKD,uBAAuB,oBAAoB;EAEhD,KAAKD,qBAAqB,IAAI,kBAC5B,KAAKD,KACL,QACA,GAAG,OAAO,GAAG,MACb,gBACA,QACA,MACA,eACF;CACF;CAEA,SAAS;EACP,OAAO,QAAQ,QAAQ,EAAC,QAAQ,KAAI,CAAC;CACvC;CAEA,MAAM;EACJ,KAAKI,cAAc,KAAKH,mBAAmB,IAAI,EAAE,MAAM,OAAM,UAAS;GACpE,IAAI,KAAKC,sBACP,MAAM,wBACJ,KAAKF,KACL,eACA,uDACF;GAEF,MAAM;EACR,CAAC;EACD,OAAO,KAAKI;CACd;CAEA,YAAkC;EAChC,OAAO,KAAKH,mBAAmB,UAAU;CAC3C;CAEA,MAAM,OAAO;EACX,KAAKA,mBAAmB,KAAK,KAAKD,GAAG;EAIrC,MAAM,KAAKI;EACX,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF"}

package/out/zero-cache/src/services/replicator/reporter/recorder.d.ts

@@ -9,5 +9,32 @@ export declare class ReplicationReportRecorder {
     readonly reportReplicaLag: (o: ObservableResult) => void;
     readonly reportTotalLag: (o: ObservableResult) => void;
     readonly reportLastTotalLag: (o: ObservableResult) => void;
+    readonly reportUpstreamClockSkew: (o: ObservableResult) => void;
 }
+/**
+ * Estimates how far the upstream database's clock is from ours, in
+ * milliseconds, positive meaning upstream is ahead.
+ *
+ * `sendTimeMs` and `receiveTimeMs` are both stamped in the
+ * replication-manager, either side of `commitTimeMs`, which upstream stamps.
+ * Assuming the upstream commit fell midway through that round trip, the
+ * difference between where it actually landed and that midpoint is the clock
+ * offset -- the same round-trip estimate NTP uses.
+ *
+ * The estimate is biased by any asymmetry between the two legs: the outbound
+ * leg is a SQL round trip, while the return leg additionally waits on a WAL
+ * flush and the walsender (which the pre-PG-17 path notes can add 50~100ms on
+ * an idle database). So treat this as a detector of gross skew -- the kind
+ * that invalidates sync.e2e_serving_lag -- and not as a sub-100ms correction.
+ *
+ * The rigorous bound, if a tighter reading is ever needed: the commit
+ * genuinely happened between send and receive, so a `commitTimeMs` outside
+ * that window proves skew of at least the amount by which it falls outside,
+ * independent of any latency assumption.
+ */
+export declare function estimateUpstreamClockSkewMs({ sendTimeMs, commitTimeMs, receiveTimeMs, }: {
+    sendTimeMs: number;
+    commitTimeMs: number;
+    receiveTimeMs: number;
+}): number;
 //# sourceMappingURL=recorder.d.ts.map

package/out/zero-cache/src/services/replicator/reporter/recorder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"recorder.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oBAAoB,CAAC;AAM1D,qBAAa,yBAAyB;;gBAIxB,EAAE,EAAE,UAAU;IAI1B,MAAM,CAAC,MAAM,EAAE,iBAAiB;IAoDhC,QAAQ,CAAC,iBAAiB,GAAI,GAAG,gBAAgB,UAK/C;IAEF,QAAQ,CAAC,gBAAgB,GAAI,GAAG,gBAAgB,UAK9C;IAEF,QAAQ,CAAC,cAAc,GAAI,GAAG,gBAAgB,UAK5C;IAEF,QAAQ,CAAC,kBAAkB,GAAI,GAAG,gBAAgB,UAKhD;CACH"}
+{"version":3,"file":"recorder.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oBAAoB,CAAC;AAM1D,qBAAa,yBAAyB;;gBAIxB,EAAE,EAAE,UAAU;IAI1B,MAAM,CAAC,MAAM,EAAE,iBAAiB;IAkEhC,QAAQ,CAAC,iBAAiB,GAAI,GAAG,gBAAgB,UAK/C;IAEF,QAAQ,CAAC,gBAAgB,GAAI,GAAG,gBAAgB,UAK9C;IAEF,QAAQ,CAAC,cAAc,GAAI,GAAG,gBAAgB,UAK5C;IAEF,QAAQ,CAAC,kBAAkB,GAAI,GAAG,gBAAgB,UAKhD;IAEF,QAAQ,CAAC,uBAAuB,GAAI,GAAG,gBAAgB,UAKrD;CACH;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,2BAA2B,CAAC,EAC1C,UAAU,EACV,YAAY,EACZ,aAAa,GACd,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,GAAG,MAAM,CAET"}

package/out/zero-cache/src/services/replicator/reporter/recorder.js

@@ -34,6 +34,10 @@ var ReplicationReportRecorder = class {
 				description: "Latency from sending the most recently received upstream replication report to its reaching the replica. This is an alias of replication.total_lag retained for dashboards that explicitly want the non-extrapolated value.",
 				unit: "millisecond"
 			}).addCallback(this.reportLastTotalLag);
+			getOrCreateGauge("replication", "upstream_clock_skew", {
+				description: "Estimated offset of the upstream database clock relative to zero-cache's. Positive means upstream is running ahead. Derived from the lag report round-trip, whose send and receive times are stamped on our clock either side of the upstream commit timestamp. This matters because sync.e2e_serving_lag subtracts an upstream commit timestamp from a local clock, so a skewed upstream clock biases it directly -- and an upstream clock running ahead biases it *low*, which looks like healthy serving rather than a broken measurement.",
+				unit: "millisecond"
+			}).addCallback(this.reportUpstreamClockSkew);
 		}
 	}
 	reportUpstreamLag = (o) => {
@@ -52,7 +56,35 @@ var ReplicationReportRecorder = class {
 		const last = this.#last?.lastTimings;
 		if (last) o.observe(last.replicateTimeMs - last.sendTimeMs);
 	};
+	reportUpstreamClockSkew = (o) => {
+		const last = this.#last?.lastTimings;
+		if (last) o.observe(estimateUpstreamClockSkewMs(last));
+	};
 };
+/**
+* Estimates how far the upstream database's clock is from ours, in
+* milliseconds, positive meaning upstream is ahead.
+*
+* `sendTimeMs` and `receiveTimeMs` are both stamped in the
+* replication-manager, either side of `commitTimeMs`, which upstream stamps.
+* Assuming the upstream commit fell midway through that round trip, the
+* difference between where it actually landed and that midpoint is the clock
+* offset -- the same round-trip estimate NTP uses.
+*
+* The estimate is biased by any asymmetry between the two legs: the outbound
+* leg is a SQL round trip, while the return leg additionally waits on a WAL
+* flush and the walsender (which the pre-PG-17 path notes can add 50~100ms on
+* an idle database). So treat this as a detector of gross skew -- the kind
+* that invalidates sync.e2e_serving_lag -- and not as a sub-100ms correction.
+*
+* The rigorous bound, if a tighter reading is ever needed: the commit
+* genuinely happened between send and receive, so a `commitTimeMs` outside
+* that window proves skew of at least the amount by which it falls outside,
+* independent of any latency assumption.
+*/
+function estimateUpstreamClockSkewMs({ sendTimeMs, commitTimeMs, receiveTimeMs }) {
+	return commitTimeMs - (sendTimeMs + receiveTimeMs) / 2;
+}
 //#endregion
 export { ReplicationReportRecorder };
 

package/out/zero-cache/src/services/replicator/reporter/recorder.js.map

@@ -1 +1 @@
-{"version":3,"file":"recorder.js","names":["#lc","#last"],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {ReplicationReport} from './report-schema.ts';\n\n// Hook for sanity checking lag reports in development.\nconst LOG_ALL_REPLICATION_REPORTS_AT_DEBUG =\n  process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === '1';\n\nexport class ReplicationReportRecorder {\n  readonly #lc: LogContext;\n  #last: ReplicationReport | null = null;\n\n  constructor(lc: LogContext) {\n    this.#lc = lc;\n  }\n\n  record(report: ReplicationReport) {\n    const first = this.#last === null;\n    this.#last = report;\n\n    const {lastTimings} = report;\n    if (lastTimings) {\n      const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;\n      if (total > 10_000) {\n        this.#lc.warn?.(`high replication lag: ${total} ms`, report);\n      } else if (total > 1_000) {\n        this.#lc.info?.(`replication lag: ${total} ms`, report);\n      }\n      if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) {\n        this.#lc.debug?.(`replication lag ${total} ms`, report);\n      }\n    }\n\n    if (first) {\n      getOrCreateGauge('replication', 'upstream_lag', {\n        description:\n          'Latency from sending an upstream replication report ' +\n          'to receiving it in the replication stream',\n        unit: 'millisecond',\n      }).addCallback(this.reportUpstreamLag);\n\n      getOrCreateGauge('replication', 'replica_lag', {\n        description:\n          'Latency from receiving an upstream replication report ' +\n          'to its reaching the replica',\n        unit: 'millisecond',\n      }).addCallback(this.reportReplicaLag);\n\n      getOrCreateGauge('replication', 'total_lag', {\n        description:\n          'Latency from sending an upstream replication report to its ' +\n          'reaching the replica. This reflects the actual measured ' +\n          'round-trip of the most recently received report and does not ' +\n          'grow when reports stop arriving.',\n        unit: 'millisecond',\n      }).addCallback(this.reportTotalLag);\n\n      getOrCreateGauge('replication', 'last_total_lag', {\n        description:\n          'Latency from sending the most recently received upstream ' +\n          'replication report to its reaching the replica. This is an alias ' +\n          'of replication.total_lag retained for dashboards that explicitly ' +\n          'want the non-extrapolated value.',\n        unit: 'millisecond',\n      }).addCallback(this.reportLastTotalLag);\n    }\n  }\n\n  readonly reportUpstreamLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.receiveTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportReplicaLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.receiveTimeMs);\n    }\n  };\n\n  readonly reportTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportLastTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n}\n"],"mappings":";;AAMA,IAAM,uCACJ,QAAQ,IAAI,8CAA8C;AAE5D,IAAa,4BAAb,MAAuC;CACrC;CACA,QAAkC;CAElC,YAAY,IAAgB;EAC1B,KAAKA,MAAM;CACb;CAEA,OAAO,QAA2B;EAChC,MAAM,QAAQ,KAAKC,UAAU;EAC7B,KAAKA,QAAQ;EAEb,MAAM,EAAC,gBAAe;EACtB,IAAI,aAAa;GACf,MAAM,QAAQ,YAAY,kBAAkB,YAAY;GACxD,IAAI,QAAQ,KACV,KAAKD,IAAI,OAAO,yBAAyB,MAAM,MAAM,MAAM;QACtD,IAAI,QAAQ,KACjB,KAAKA,IAAI,OAAO,oBAAoB,MAAM,MAAM,MAAM;GAExD,IAAI,sCACF,KAAKA,IAAI,QAAQ,mBAAmB,MAAM,MAAM,MAAM;EAE1D;EAEA,IAAI,OAAO;GACT,iBAAiB,eAAe,gBAAgB;IAC9C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,iBAAiB;GAErC,iBAAiB,eAAe,eAAe;IAC7C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,gBAAgB;GAEpC,iBAAiB,eAAe,aAAa;IAC3C,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,cAAc;GAElC,iBAAiB,eAAe,kBAAkB;IAChD,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,kBAAkB;EACxC;CACF;CAEA,qBAA8B,MAAwB;EACpD,MAAM,OAAO,KAAKC,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,gBAAgB,KAAK,UAAU;CAElD;CAEA,oBAA6B,MAAwB;EACnD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,aAAa;CAEvD;CAEA,kBAA2B,MAAwB;EACjD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,sBAA+B,MAAwB;EACrD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;AACF"}
+{"version":3,"file":"recorder.js","names":["#lc","#last"],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {ReplicationReport} from './report-schema.ts';\n\n// Hook for sanity checking lag reports in development.\nconst LOG_ALL_REPLICATION_REPORTS_AT_DEBUG =\n  process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === '1';\n\nexport class ReplicationReportRecorder {\n  readonly #lc: LogContext;\n  #last: ReplicationReport | null = null;\n\n  constructor(lc: LogContext) {\n    this.#lc = lc;\n  }\n\n  record(report: ReplicationReport) {\n    const first = this.#last === null;\n    this.#last = report;\n\n    const {lastTimings} = report;\n    if (lastTimings) {\n      const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;\n      if (total > 10_000) {\n        this.#lc.warn?.(`high replication lag: ${total} ms`, report);\n      } else if (total > 1_000) {\n        this.#lc.info?.(`replication lag: ${total} ms`, report);\n      }\n      if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) {\n        this.#lc.debug?.(`replication lag ${total} ms`, report);\n      }\n    }\n\n    if (first) {\n      getOrCreateGauge('replication', 'upstream_lag', {\n        description:\n          'Latency from sending an upstream replication report ' +\n          'to receiving it in the replication stream',\n        unit: 'millisecond',\n      }).addCallback(this.reportUpstreamLag);\n\n      getOrCreateGauge('replication', 'replica_lag', {\n        description:\n          'Latency from receiving an upstream replication report ' +\n          'to its reaching the replica',\n        unit: 'millisecond',\n      }).addCallback(this.reportReplicaLag);\n\n      getOrCreateGauge('replication', 'total_lag', {\n        description:\n          'Latency from sending an upstream replication report to its ' +\n          'reaching the replica. This reflects the actual measured ' +\n          'round-trip of the most recently received report and does not ' +\n          'grow when reports stop arriving.',\n        unit: 'millisecond',\n      }).addCallback(this.reportTotalLag);\n\n      getOrCreateGauge('replication', 'last_total_lag', {\n        description:\n          'Latency from sending the most recently received upstream ' +\n          'replication report to its reaching the replica. This is an alias ' +\n          'of replication.total_lag retained for dashboards that explicitly ' +\n          'want the non-extrapolated value.',\n        unit: 'millisecond',\n      }).addCallback(this.reportLastTotalLag);\n\n      getOrCreateGauge('replication', 'upstream_clock_skew', {\n        description:\n          'Estimated offset of the upstream database clock relative to ' +\n          \"zero-cache's. Positive means upstream is running ahead. \" +\n          'Derived from the lag report round-trip, whose send and receive ' +\n          'times are stamped on our clock either side of the upstream ' +\n          'commit timestamp. This matters because ' +\n          'sync.e2e_serving_lag subtracts an upstream commit timestamp from ' +\n          'a local clock, so a skewed upstream clock biases it directly -- ' +\n          'and an upstream clock running ahead biases it *low*, which looks ' +\n          'like healthy serving rather than a broken measurement.',\n        unit: 'millisecond',\n      }).addCallback(this.reportUpstreamClockSkew);\n    }\n  }\n\n  readonly reportUpstreamLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.receiveTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportReplicaLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.receiveTimeMs);\n    }\n  };\n\n  readonly reportTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportLastTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportUpstreamClockSkew = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(estimateUpstreamClockSkewMs(last));\n    }\n  };\n}\n\n/**\n * Estimates how far the upstream database's clock is from ours, in\n * milliseconds, positive meaning upstream is ahead.\n *\n * `sendTimeMs` and `receiveTimeMs` are both stamped in the\n * replication-manager, either side of `commitTimeMs`, which upstream stamps.\n * Assuming the upstream commit fell midway through that round trip, the\n * difference between where it actually landed and that midpoint is the clock\n * offset -- the same round-trip estimate NTP uses.\n *\n * The estimate is biased by any asymmetry between the two legs: the outbound\n * leg is a SQL round trip, while the return leg additionally waits on a WAL\n * flush and the walsender (which the pre-PG-17 path notes can add 50~100ms on\n * an idle database). So treat this as a detector of gross skew -- the kind\n * that invalidates sync.e2e_serving_lag -- and not as a sub-100ms correction.\n *\n * The rigorous bound, if a tighter reading is ever needed: the commit\n * genuinely happened between send and receive, so a `commitTimeMs` outside\n * that window proves skew of at least the amount by which it falls outside,\n * independent of any latency assumption.\n */\nexport function estimateUpstreamClockSkewMs({\n  sendTimeMs,\n  commitTimeMs,\n  receiveTimeMs,\n}: {\n  sendTimeMs: number;\n  commitTimeMs: number;\n  receiveTimeMs: number;\n}): number {\n  return commitTimeMs - (sendTimeMs + receiveTimeMs) / 2;\n}\n"],"mappings":";;AAMA,IAAM,uCACJ,QAAQ,IAAI,8CAA8C;AAE5D,IAAa,4BAAb,MAAuC;CACrC;CACA,QAAkC;CAElC,YAAY,IAAgB;EAC1B,KAAKA,MAAM;CACb;CAEA,OAAO,QAA2B;EAChC,MAAM,QAAQ,KAAKC,UAAU;EAC7B,KAAKA,QAAQ;EAEb,MAAM,EAAC,gBAAe;EACtB,IAAI,aAAa;GACf,MAAM,QAAQ,YAAY,kBAAkB,YAAY;GACxD,IAAI,QAAQ,KACV,KAAKD,IAAI,OAAO,yBAAyB,MAAM,MAAM,MAAM;QACtD,IAAI,QAAQ,KACjB,KAAKA,IAAI,OAAO,oBAAoB,MAAM,MAAM,MAAM;GAExD,IAAI,sCACF,KAAKA,IAAI,QAAQ,mBAAmB,MAAM,MAAM,MAAM;EAE1D;EAEA,IAAI,OAAO;GACT,iBAAiB,eAAe,gBAAgB;IAC9C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,iBAAiB;GAErC,iBAAiB,eAAe,eAAe;IAC7C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,gBAAgB;GAEpC,iBAAiB,eAAe,aAAa;IAC3C,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,cAAc;GAElC,iBAAiB,eAAe,kBAAkB;IAChD,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,kBAAkB;GAEtC,iBAAiB,eAAe,uBAAuB;IACrD,aACE;IASF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,uBAAuB;EAC7C;CACF;CAEA,qBAA8B,MAAwB;EACpD,MAAM,OAAO,KAAKC,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,gBAAgB,KAAK,UAAU;CAElD;CAEA,oBAA6B,MAAwB;EACnD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,aAAa;CAEvD;CAEA,kBAA2B,MAAwB;EACjD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,sBAA+B,MAAwB;EACrD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,2BAAoC,MAAwB;EAC1D,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,4BAA4B,IAAI,CAAC;CAE/C;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,SAAgB,4BAA4B,EAC1C,YACA,cACA,iBAKS;CACT,OAAO,gBAAgB,aAAa,iBAAiB;AACvD"}

package/out/zero-cache/src/services/replicator/reporter/report-schema.d.ts

@@ -21,8 +21,8 @@ export declare const replicationTimingsSchema: v.ObjectType<Omit<{
 }, undefined>;
 export declare const replicationReportSchema: v.ObjectType<{
     lastTimings: v.Optional<{
-        sendTimeMs: number;
         commitTimeMs: number;
+        sendTimeMs: number;
         receiveTimeMs: number;
         replicateTimeMs: number;
     }>;

package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.d.ts

@@ -0,0 +1,44 @@
+import type { ReplicaState } from '../replicator/replicator.ts';
+export type PendingUpstreamCommit = {
+    readonly watermark: string;
+    readonly commitTimeMs: number;
+};
+/**
+ * Pairs the upstream commit timestamps carried by `version-ready`
+ * notifications with the moment the corresponding change is poked to clients,
+ * yielding the end-to-end serving lag.
+ *
+ * This measures completion, not backlog: an observation is produced only when
+ * work actually reaches clients, so the resulting histogram is a latency
+ * distribution rather than a periodic snapshot of how far behind things are.
+ * A ViewSyncer that is stuck contributes nothing until it recovers, instead of
+ * re-reporting its age on every sample tick.
+ */
+export declare class E2EServingLagTracker {
+    #private;
+    get pending(): PendingUpstreamCommit | null;
+    /**
+     * Records the upstream commit behind a `version-ready` notification.
+     *
+     * Notifications coalesce when the ViewSyncer is busy, so one state may stand
+     * in for several commits. The *oldest* commit time is kept, since it bounds
+     * the lag of everything the notification subsumed, while the watermark
+     * advances to the newest — that is the one that must be served for all of
+     * the subsumed commits to have been delivered.
+     */
+    onVersionReady({ watermark, upstreamCommitTimeMs }: ReplicaState): void;
+    /**
+     * Called once a version has been poked to clients.
+     *
+     * @return the observation to record, or `null` if the served version does
+     *     not yet cover an outstanding upstream commit.
+     */
+    onVersionServed(servedVersion: string, nowMs: number): Observation | null;
+}
+export type Observation = {
+    /** End-to-end lag in milliseconds, clamped to be non-negative. */
+    readonly lagMs: number;
+    /** Whether the clamp was applied, i.e. the raw measurement was negative. */
+    readonly clamped: boolean;
+};
+//# sourceMappingURL=e2e-serving-lag.d.ts.map

package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"e2e-serving-lag.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/e2e-serving-lag.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,oBAAoB;;IAG/B,IAAI,OAAO,IAAI,qBAAqB,GAAG,IAAI,CAE1C;IAED;;;;;;;;OAQG;IACH,cAAc,CAAC,EAAC,SAAS,EAAE,oBAAoB,EAAC,EAAE,YAAY,GAAG,IAAI;IAcrE;;;;;OAKG;IACH,eAAe,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI;CAoB1E;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kEAAkE;IAClE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,4EAA4E;IAC5E,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B,CAAC"}

package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.js

@@ -0,0 +1,59 @@
+//#region ../zero-cache/src/services/view-syncer/e2e-serving-lag.ts
+/**
+* Pairs the upstream commit timestamps carried by `version-ready`
+* notifications with the moment the corresponding change is poked to clients,
+* yielding the end-to-end serving lag.
+*
+* This measures completion, not backlog: an observation is produced only when
+* work actually reaches clients, so the resulting histogram is a latency
+* distribution rather than a periodic snapshot of how far behind things are.
+* A ViewSyncer that is stuck contributes nothing until it recovers, instead of
+* re-reporting its age on every sample tick.
+*/
+var E2EServingLagTracker = class {
+	#pending = null;
+	get pending() {
+		return this.#pending;
+	}
+	/**
+	* Records the upstream commit behind a `version-ready` notification.
+	*
+	* Notifications coalesce when the ViewSyncer is busy, so one state may stand
+	* in for several commits. The *oldest* commit time is kept, since it bounds
+	* the lag of everything the notification subsumed, while the watermark
+	* advances to the newest — that is the one that must be served for all of
+	* the subsumed commits to have been delivered.
+	*/
+	onVersionReady({ watermark, upstreamCommitTimeMs }) {
+		if (watermark === void 0 || upstreamCommitTimeMs === void 0) return;
+		const pending = this.#pending;
+		this.#pending = {
+			watermark,
+			commitTimeMs: pending === null ? upstreamCommitTimeMs : Math.min(pending.commitTimeMs, upstreamCommitTimeMs)
+		};
+	}
+	/**
+	* Called once a version has been poked to clients.
+	*
+	* @return the observation to record, or `null` if the served version does
+	*     not yet cover an outstanding upstream commit.
+	*/
+	onVersionServed(servedVersion, nowMs) {
+		const pending = this.#pending;
+		if (pending === null || servedVersion < pending.watermark) return null;
+		this.#pending = null;
+		const lagMs = nowMs - pending.commitTimeMs;
+		if (lagMs >= 0) return {
+			lagMs,
+			clamped: false
+		};
+		return {
+			lagMs: 0,
+			clamped: true
+		};
+	}
+};
+//#endregion
+export { E2EServingLagTracker };
+
+//# sourceMappingURL=e2e-serving-lag.js.map

package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"e2e-serving-lag.js","names":["#pending"],"sources":["../../../../../../zero-cache/src/services/view-syncer/e2e-serving-lag.ts"],"sourcesContent":["import type {ReplicaState} from '../replicator/replicator.ts';\n\nexport type PendingUpstreamCommit = {\n  readonly watermark: string;\n  readonly commitTimeMs: number;\n};\n\n/**\n * Pairs the upstream commit timestamps carried by `version-ready`\n * notifications with the moment the corresponding change is poked to clients,\n * yielding the end-to-end serving lag.\n *\n * This measures completion, not backlog: an observation is produced only when\n * work actually reaches clients, so the resulting histogram is a latency\n * distribution rather than a periodic snapshot of how far behind things are.\n * A ViewSyncer that is stuck contributes nothing until it recovers, instead of\n * re-reporting its age on every sample tick.\n */\nexport class E2EServingLagTracker {\n  #pending: PendingUpstreamCommit | null = null;\n\n  get pending(): PendingUpstreamCommit | null {\n    return this.#pending;\n  }\n\n  /**\n   * Records the upstream commit behind a `version-ready` notification.\n   *\n   * Notifications coalesce when the ViewSyncer is busy, so one state may stand\n   * in for several commits. The *oldest* commit time is kept, since it bounds\n   * the lag of everything the notification subsumed, while the watermark\n   * advances to the newest — that is the one that must be served for all of\n   * the subsumed commits to have been delivered.\n   */\n  onVersionReady({watermark, upstreamCommitTimeMs}: ReplicaState): void {\n    if (watermark === undefined || upstreamCommitTimeMs === undefined) {\n      return;\n    }\n    const pending = this.#pending;\n    this.#pending = {\n      watermark,\n      commitTimeMs:\n        pending === null\n          ? upstreamCommitTimeMs\n          : Math.min(pending.commitTimeMs, upstreamCommitTimeMs),\n    };\n  }\n\n  /**\n   * Called once a version has been poked to clients.\n   *\n   * @return the observation to record, or `null` if the served version does\n   *     not yet cover an outstanding upstream commit.\n   */\n  onVersionServed(servedVersion: string, nowMs: number): Observation | null {\n    const pending = this.#pending;\n    if (pending === null || servedVersion < pending.watermark) {\n      return null;\n    }\n    this.#pending = null;\n\n    const lagMs = nowMs - pending.commitTimeMs;\n    if (lagMs >= 0) {\n      return {lagMs, clamped: false};\n    }\n    // The commit time is on the upstream database's clock while `nowMs` is\n    // local, so a negative duration means upstream's clock is running ahead of\n    // ours by more than the entire pipeline latency. Clamp, because a negative\n    // value would corrupt the histogram's sum -- but report the clamp, since\n    // it is proof of gross clock skew, and skew in this direction biases the\n    // metric *low*, which reads as healthy serving rather than as a broken\n    // measurement. See replication.upstream_clock_skew for the magnitude.\n    return {lagMs: 0, clamped: true};\n  }\n}\n\nexport type Observation = {\n  /** End-to-end lag in milliseconds, clamped to be non-negative. */\n  readonly lagMs: number;\n  /** Whether the clamp was applied, i.e. the raw measurement was negative. */\n  readonly clamped: boolean;\n};\n"],"mappings":";;;;;;;;;;;;AAkBA,IAAa,uBAAb,MAAkC;CAChC,WAAyC;CAEzC,IAAI,UAAwC;EAC1C,OAAO,KAAKA;CACd;;;;;;;;;;CAWA,eAAe,EAAC,WAAW,wBAA2C;EACpE,IAAI,cAAc,KAAA,KAAa,yBAAyB,KAAA,GACtD;EAEF,MAAM,UAAU,KAAKA;EACrB,KAAKA,WAAW;GACd;GACA,cACE,YAAY,OACR,uBACA,KAAK,IAAI,QAAQ,cAAc,oBAAoB;EAC3D;CACF;;;;;;;CAQA,gBAAgB,eAAuB,OAAmC;EACxE,MAAM,UAAU,KAAKA;EACrB,IAAI,YAAY,QAAQ,gBAAgB,QAAQ,WAC9C,OAAO;EAET,KAAKA,WAAW;EAEhB,MAAM,QAAQ,QAAQ,QAAQ;EAC9B,IAAI,SAAS,GACX,OAAO;GAAC;GAAO,SAAS;EAAK;EAS/B,OAAO;GAAC,OAAO;GAAG,SAAS;EAAI;CACjC;AACF"}

package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"view-syncer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/view-syncer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,2BAA2B,EAAC,MAAM,yDAAyD,CAAC;AACzG,OAAO,KAAK,EAEV,qBAAqB,EACtB,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,iDAAiD,CAAC;AAC1F,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAQtE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAGpF,OAAO,EAAkB,KAAK,IAAI,EAAC,MAAM,oBAAoB,CAAC;AAK9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EACV,sBAAsB,EAEvB,MAAM,yCAAyC,CAAC;AAOjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAM1E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAQxD,OAAO,KAAK,EAEV,wBAAwB,EACxB,kBAAkB,EAEnB,MAAM,iCAAiC,CAAC;AAUzC,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAuCzD,MAAM,WAAW,UAAU;IACzB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC,CAAC;IAEtB,oBAAoB,CAClB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,aAAa,CACX,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAErB,OAAO,CAAC,QAAQ,EAAE,kBAAkB,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,UAAU,CACR,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,iBAAiB,EACtB,mBAAmB,EAAE,OAAO,GAC3B,OAAO,CAAC,IAAI,CAAC,CAAC;IAGjB,kBAAkB,EAAE,wBAAwB,CAAC;IAE7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,MAAM,WAAW,GAAG,kBAAkB,GAAG;IAC7C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC;CACjC,CAAC;AAmBF,KAAK,UAAU,GAAG,CAChB,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAChC,KAAK,CAAC,EAAE,MAAM,KACX,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAIvC,qBAAa,iBAAkB,YAAW,UAAU,EAAE,oBAAoB;;IACxE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,SAAc;IAGlC,QAAQ,CAAC,kBAAkB,EAAE,wBAAwB,CAAC;gBA+JpD,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,UAAU,EACjB,cAAc,EAAE,cAAc,EAC9B,cAAc,EAAE,YAAY,CAAC,YAAY,CAAC,EAC1C,gBAAgB,EAAE,gBAAgB,EAClC,oBAAoB,EAAE,MAAM,EAC5B,iBAAiB,EAAE,iBAAiB,EACpC,kBAAkB,EAAE,wBAAwB,EAC5C,sBAAsB,EAAE,sBAAsB,GAAG,SAAS,EAC1D,aAAa,EAAE,CAAC,CAAC,EACf,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,KACjB,OAAO,CAAC,CAAC,CAAC,EACf,WAAW,SAAuB,EAClC,YAAY,GAAE,UAAwC;IAiGxD,UAAU,IAAI,OAAO,CAAC,aAAa,GAAG,UAAU,CAAC;IAO3C,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAgI1B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAEjC;IAED,IAAI,kBAAkB,IAAI,OAAO,CAKhC;IAQD;;;;;;;;OAQG;IACH,SAAS,IAAI,OAAO;IAkKpB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC;IAuGf,oBAAoB,CACxB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC;IAiBV,UAAU,CACd,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,iBAAiB,EACtB,mBAAmB,EAAE,OAAO,GAC3B,OAAO,CAAC,IAAI,CAAC;IAsCV,aAAa,CACjB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC;IAikDd,OAAO,CACX,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,IAAI,CAAC;IAmKhB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAgCrB;;;OAGG;IACH,eAAe;CAGhB;AAsGD,qBAAa,cAAc;;gBAKb,EAAE,EAAE,UAAU;IAIpB,KAAK;IAOX,oBAAoB;IAMd,YAAY,CAAC,cAAc,CAAC,EAAE,MAAM;IAW1C,UAAU;IAWV,sCAAsC;IACtC,IAAI,IAAI,MAAM;IAKd;;;OAGG;IACH,YAAY,IAAI,MAAM;CAKvB"}
+{"version":3,"file":"view-syncer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/view-syncer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,2BAA2B,EAAC,MAAM,yDAAyD,CAAC;AACzG,OAAO,KAAK,EAEV,qBAAqB,EACtB,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,iDAAiD,CAAC;AAC1F,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAQtE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAGpF,OAAO,EAAkB,KAAK,IAAI,EAAC,MAAM,oBAAoB,CAAC;AAK9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EACV,sBAAsB,EAEvB,MAAM,yCAAyC,CAAC;AAOjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAM1E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAQxD,OAAO,KAAK,EAEV,wBAAwB,EACxB,kBAAkB,EAEnB,MAAM,iCAAiC,CAAC;AAUzC,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAG7D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAuCzD,MAAM,WAAW,UAAU;IACzB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC,CAAC;IAEtB,oBAAoB,CAClB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,aAAa,CACX,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAErB,OAAO,CAAC,QAAQ,EAAE,kBAAkB,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,UAAU,CACR,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,iBAAiB,EACtB,mBAAmB,EAAE,OAAO,GAC3B,OAAO,CAAC,IAAI,CAAC,CAAC;IAGjB,kBAAkB,EAAE,wBAAwB,CAAC;IAE7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,MAAM,WAAW,GAAG,kBAAkB,GAAG;IAC7C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC;CACjC,CAAC;AAmBF,KAAK,UAAU,GAAG,CAChB,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAChC,KAAK,CAAC,EAAE,MAAM,KACX,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAIvC,qBAAa,iBAAkB,YAAW,UAAU,EAAE,oBAAoB;;IACxE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,SAAc;IAGlC,QAAQ,CAAC,kBAAkB,EAAE,wBAAwB,CAAC;gBA0LpD,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,UAAU,EACjB,cAAc,EAAE,cAAc,EAC9B,cAAc,EAAE,YAAY,CAAC,YAAY,CAAC,EAC1C,gBAAgB,EAAE,gBAAgB,EAClC,oBAAoB,EAAE,MAAM,EAC5B,iBAAiB,EAAE,iBAAiB,EACpC,kBAAkB,EAAE,wBAAwB,EAC5C,sBAAsB,EAAE,sBAAsB,GAAG,SAAS,EAC1D,aAAa,EAAE,CAAC,CAAC,EACf,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,KACjB,OAAO,CAAC,CAAC,CAAC,EACf,WAAW,SAAuB,EAClC,YAAY,GAAE,UAAwC;IAiGxD,UAAU,IAAI,OAAO,CAAC,aAAa,GAAG,UAAU,CAAC;IAO3C,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAkI1B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAEjC;IAED,IAAI,kBAAkB,IAAI,OAAO,CAKhC;IAkBD;;;;;;;;OAQG;IACH,SAAS,IAAI,OAAO;IAkKpB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC;IAuGf,oBAAoB,CACxB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC;IAiBV,UAAU,CACd,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,iBAAiB,EACtB,mBAAmB,EAAE,OAAO,GAC3B,OAAO,CAAC,IAAI,CAAC;IAsCV,aAAa,CACjB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC;IAikDd,OAAO,CACX,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,IAAI,CAAC;IAmKhB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAgCrB;;;OAGG;IACH,eAAe;CAGhB;AAsGD,qBAAa,cAAc;;gBAKb,EAAE,EAAE,UAAU;IAIpB,KAAK;IAOX,oBAAoB;IAMd,YAAY,CAAC,cAAc,CAAC,EAAE,MAAM;IAW1C,UAAU;IAWV,sCAAsC;IACtC,IAAI,IAAI,MAAM;IAKd;;;OAGG;IACH,YAAY,IAAI,MAAM;CAKvB"}

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -25,6 +25,7 @@ import { ClientHandler, startPoke } from "./client-handler.js";
 import { tracer } from "./tracer.js";
 import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
 import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, nextEvictionTime } from "./cvr.js";
+import { E2EServingLagTracker } from "./e2e-serving-lag.js";
 import { handleInspect } from "./inspect-handler.js";
 import { QueryCoveringIndex } from "./query-covering.js";
 import { resolver } from "@rocicorp/resolver";
@@ -90,6 +91,7 @@ var ViewSyncerService = class {
 	#cvr;
 	#pipelinesSynced = false;
 	#servedVersion = null;
+	#e2eServingLagTracker = new E2EServingLagTracker();
 	#expiredQueriesTimer = 0;
 	#authMaintenanceTimer = 0;
 	#setTimeout;
@@ -102,6 +104,14 @@ var ViewSyncerService = class {
 		description: "Time from ViewSyncer query sync requiring hydration to output for a client group. Includes query transformation, query materialization, CVR flush, catchup, and pokeEnd.",
 		unit: "s"
 	});
+	#e2eServingLag = getOrCreateNativeHistogram("sync", "e2e_serving_lag", {
+		description: "End-to-end lag from upstream commit to ViewSyncer output. Spans the whole pipeline: the upstream transaction commit, replication to the replica, IVM advancement, CVR flush, and pokeEnd. Recorded once per served version, not sampled, so each observation is the completion latency of real replicated work.",
+		unit: "s"
+	});
+	#e2eServingLagClamps = getOrCreateCounter("sync", "e2e_serving_lag_clamps", {
+		description: "Observations of sync.e2e_serving_lag that came out negative and were clamped to zero. Non-zero means the upstream database clock is running ahead of this pod by more than the entire pipeline latency, so sync.e2e_serving_lag is biased low and reads healthier than reality. See replication.upstream_clock_skew for the magnitude.",
+		unit: "{observation}"
+	});
 	#transactionAdvanceTime = getOrCreateLatencyHistogram("sync", "advance-time", "Time to advance all queries for a given client group after applying a new transaction to the replica.");
 	#queryTransformations = getOrCreateCounter("sync", "query.transformations", "Number of query transformations performed");
 	#queryTransformationTime = getOrCreateLatencyHistogram("sync", "query.transformation-time", "Time to transform custom queries via API server.");
@@ -186,12 +196,14 @@ var ViewSyncerService = class {
 				this.#lc.debug?.(`draining view-syncer ${this.id} before running`);
 				this.stop();
 			}
-			for await (const { state } of this.#stateChanges) {
+			for await (const replicaState of this.#stateChanges) {
+				const { state } = replicaState;
 				if (this.#drainCoordinator.shouldDrain()) {
 					this.#lc.debug?.(`draining view-syncer ${this.id} (elective)`);
 					break;
 				}
 				assert(state === "version-ready", "state should be version-ready");
+				this.#e2eServingLagTracker.onVersionReady(replicaState);
 				await this.#runInLockWithCVR(async (lc, cvr) => {
 					const clientSchema = must(cvr.clientSchema, "cvr.clientSchema missing after initialization");
 					if (!this.#pipelines.initialized()) this.#pipelines.init(clientSchema);
@@ -258,6 +270,11 @@ var ViewSyncerService = class {
 	}
 	#markVersionServed(version) {
 		this.#servedVersion = version.stateVersion;
+		const observation = this.#e2eServingLagTracker.onVersionServed(version.stateVersion, Date.now());
+		if (observation !== null) {
+			this.#e2eServingLag.recordMs(observation.lagMs);
+			if (observation.clamped) this.#e2eServingLagClamps.add(1);
+		}
 	}
 	#keepAliveUntil = 0;
 	/**