@rocicorp/zero 1.9.0-canary.2 → 1.9.0-canary.4

package/out/zero-cache/src/services/replicator/reporter/recorder.js

@@ -34,6 +34,10 @@ var ReplicationReportRecorder = class {
 				description: "Latency from sending the most recently received upstream replication report to its reaching the replica. This is an alias of replication.total_lag retained for dashboards that explicitly want the non-extrapolated value.",
 				unit: "millisecond"
 			}).addCallback(this.reportLastTotalLag);
+			getOrCreateGauge("replication", "upstream_clock_skew", {
+				description: "Estimated offset of the upstream database clock relative to zero-cache's. Positive means upstream is running ahead. Derived from the lag report round-trip, whose send and receive times are stamped on our clock either side of the upstream commit timestamp. This matters because sync.e2e_serving_lag subtracts an upstream commit timestamp from a local clock, so a skewed upstream clock biases it directly -- and an upstream clock running ahead biases it *low*, which looks like healthy serving rather than a broken measurement.",
+				unit: "millisecond"
+			}).addCallback(this.reportUpstreamClockSkew);
 		}
 	}
 	reportUpstreamLag = (o) => {
@@ -52,7 +56,35 @@ var ReplicationReportRecorder = class {
 		const last = this.#last?.lastTimings;
 		if (last) o.observe(last.replicateTimeMs - last.sendTimeMs);
 	};
+	reportUpstreamClockSkew = (o) => {
+		const last = this.#last?.lastTimings;
+		if (last) o.observe(estimateUpstreamClockSkewMs(last));
+	};
 };
+/**
+* Estimates how far the upstream database's clock is from ours, in
+* milliseconds, positive meaning upstream is ahead.
+*
+* `sendTimeMs` and `receiveTimeMs` are both stamped in the
+* replication-manager, either side of `commitTimeMs`, which upstream stamps.
+* Assuming the upstream commit fell midway through that round trip, the
+* difference between where it actually landed and that midpoint is the clock
+* offset -- the same round-trip estimate NTP uses.
+*
+* The estimate is biased by any asymmetry between the two legs: the outbound
+* leg is a SQL round trip, while the return leg additionally waits on a WAL
+* flush and the walsender (which the pre-PG-17 path notes can add 50~100ms on
+* an idle database). So treat this as a detector of gross skew -- the kind
+* that invalidates sync.e2e_serving_lag -- and not as a sub-100ms correction.
+*
+* The rigorous bound, if a tighter reading is ever needed: the commit
+* genuinely happened between send and receive, so a `commitTimeMs` outside
+* that window proves skew of at least the amount by which it falls outside,
+* independent of any latency assumption.
+*/
+function estimateUpstreamClockSkewMs({ sendTimeMs, commitTimeMs, receiveTimeMs }) {
+	return commitTimeMs - (sendTimeMs + receiveTimeMs) / 2;
+}
 //#endregion
 export { ReplicationReportRecorder };
 

package/out/zero-cache/src/services/replicator/reporter/recorder.js.map

@@ -1 +1 @@
-{"version":3,"file":"recorder.js","names":["#lc","#last"],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {ReplicationReport} from './report-schema.ts';\n\n// Hook for sanity checking lag reports in development.\nconst LOG_ALL_REPLICATION_REPORTS_AT_DEBUG =\n  process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === '1';\n\nexport class ReplicationReportRecorder {\n  readonly #lc: LogContext;\n  #last: ReplicationReport | null = null;\n\n  constructor(lc: LogContext) {\n    this.#lc = lc;\n  }\n\n  record(report: ReplicationReport) {\n    const first = this.#last === null;\n    this.#last = report;\n\n    const {lastTimings} = report;\n    if (lastTimings) {\n      const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;\n      if (total > 10_000) {\n        this.#lc.warn?.(`high replication lag: ${total} ms`, report);\n      } else if (total > 1_000) {\n        this.#lc.info?.(`replication lag: ${total} ms`, report);\n      }\n      if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) {\n        this.#lc.debug?.(`replication lag ${total} ms`, report);\n      }\n    }\n\n    if (first) {\n      getOrCreateGauge('replication', 'upstream_lag', {\n        description:\n          'Latency from sending an upstream replication report ' +\n          'to receiving it in the replication stream',\n        unit: 'millisecond',\n      }).addCallback(this.reportUpstreamLag);\n\n      getOrCreateGauge('replication', 'replica_lag', {\n        description:\n          'Latency from receiving an upstream replication report ' +\n          'to its reaching the replica',\n        unit: 'millisecond',\n      }).addCallback(this.reportReplicaLag);\n\n      getOrCreateGauge('replication', 'total_lag', {\n        description:\n          'Latency from sending an upstream replication report to its ' +\n          'reaching the replica. This reflects the actual measured ' +\n          'round-trip of the most recently received report and does not ' +\n          'grow when reports stop arriving.',\n        unit: 'millisecond',\n      }).addCallback(this.reportTotalLag);\n\n      getOrCreateGauge('replication', 'last_total_lag', {\n        description:\n          'Latency from sending the most recently received upstream ' +\n          'replication report to its reaching the replica. This is an alias ' +\n          'of replication.total_lag retained for dashboards that explicitly ' +\n          'want the non-extrapolated value.',\n        unit: 'millisecond',\n      }).addCallback(this.reportLastTotalLag);\n    }\n  }\n\n  readonly reportUpstreamLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.receiveTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportReplicaLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.receiveTimeMs);\n    }\n  };\n\n  readonly reportTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportLastTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n}\n"],"mappings":";;AAMA,IAAM,uCACJ,QAAQ,IAAI,8CAA8C;AAE5D,IAAa,4BAAb,MAAuC;CACrC;CACA,QAAkC;CAElC,YAAY,IAAgB;EAC1B,KAAKA,MAAM;CACb;CAEA,OAAO,QAA2B;EAChC,MAAM,QAAQ,KAAKC,UAAU;EAC7B,KAAKA,QAAQ;EAEb,MAAM,EAAC,gBAAe;EACtB,IAAI,aAAa;GACf,MAAM,QAAQ,YAAY,kBAAkB,YAAY;GACxD,IAAI,QAAQ,KACV,KAAKD,IAAI,OAAO,yBAAyB,MAAM,MAAM,MAAM;QACtD,IAAI,QAAQ,KACjB,KAAKA,IAAI,OAAO,oBAAoB,MAAM,MAAM,MAAM;GAExD,IAAI,sCACF,KAAKA,IAAI,QAAQ,mBAAmB,MAAM,MAAM,MAAM;EAE1D;EAEA,IAAI,OAAO;GACT,iBAAiB,eAAe,gBAAgB;IAC9C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,iBAAiB;GAErC,iBAAiB,eAAe,eAAe;IAC7C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,gBAAgB;GAEpC,iBAAiB,eAAe,aAAa;IAC3C,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,cAAc;GAElC,iBAAiB,eAAe,kBAAkB;IAChD,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,kBAAkB;EACxC;CACF;CAEA,qBAA8B,MAAwB;EACpD,MAAM,OAAO,KAAKC,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,gBAAgB,KAAK,UAAU;CAElD;CAEA,oBAA6B,MAAwB;EACnD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,aAAa;CAEvD;CAEA,kBAA2B,MAAwB;EACjD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,sBAA+B,MAAwB;EACrD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;AACF"}
+{"version":3,"file":"recorder.js","names":["#lc","#last"],"sources":["../../../../../../../zero-cache/src/services/replicator/reporter/recorder.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {ReplicationReport} from './report-schema.ts';\n\n// Hook for sanity checking lag reports in development.\nconst LOG_ALL_REPLICATION_REPORTS_AT_DEBUG =\n  process.env.ZERO_LOG_ALL_REPLICATION_REPORTS_AT_DEBUG === '1';\n\nexport class ReplicationReportRecorder {\n  readonly #lc: LogContext;\n  #last: ReplicationReport | null = null;\n\n  constructor(lc: LogContext) {\n    this.#lc = lc;\n  }\n\n  record(report: ReplicationReport) {\n    const first = this.#last === null;\n    this.#last = report;\n\n    const {lastTimings} = report;\n    if (lastTimings) {\n      const total = lastTimings.replicateTimeMs - lastTimings.sendTimeMs;\n      if (total > 10_000) {\n        this.#lc.warn?.(`high replication lag: ${total} ms`, report);\n      } else if (total > 1_000) {\n        this.#lc.info?.(`replication lag: ${total} ms`, report);\n      }\n      if (LOG_ALL_REPLICATION_REPORTS_AT_DEBUG) {\n        this.#lc.debug?.(`replication lag ${total} ms`, report);\n      }\n    }\n\n    if (first) {\n      getOrCreateGauge('replication', 'upstream_lag', {\n        description:\n          'Latency from sending an upstream replication report ' +\n          'to receiving it in the replication stream',\n        unit: 'millisecond',\n      }).addCallback(this.reportUpstreamLag);\n\n      getOrCreateGauge('replication', 'replica_lag', {\n        description:\n          'Latency from receiving an upstream replication report ' +\n          'to its reaching the replica',\n        unit: 'millisecond',\n      }).addCallback(this.reportReplicaLag);\n\n      getOrCreateGauge('replication', 'total_lag', {\n        description:\n          'Latency from sending an upstream replication report to its ' +\n          'reaching the replica. This reflects the actual measured ' +\n          'round-trip of the most recently received report and does not ' +\n          'grow when reports stop arriving.',\n        unit: 'millisecond',\n      }).addCallback(this.reportTotalLag);\n\n      getOrCreateGauge('replication', 'last_total_lag', {\n        description:\n          'Latency from sending the most recently received upstream ' +\n          'replication report to its reaching the replica. This is an alias ' +\n          'of replication.total_lag retained for dashboards that explicitly ' +\n          'want the non-extrapolated value.',\n        unit: 'millisecond',\n      }).addCallback(this.reportLastTotalLag);\n\n      getOrCreateGauge('replication', 'upstream_clock_skew', {\n        description:\n          'Estimated offset of the upstream database clock relative to ' +\n          \"zero-cache's. Positive means upstream is running ahead. \" +\n          'Derived from the lag report round-trip, whose send and receive ' +\n          'times are stamped on our clock either side of the upstream ' +\n          'commit timestamp. This matters because ' +\n          'sync.e2e_serving_lag subtracts an upstream commit timestamp from ' +\n          'a local clock, so a skewed upstream clock biases it directly -- ' +\n          'and an upstream clock running ahead biases it *low*, which looks ' +\n          'like healthy serving rather than a broken measurement.',\n        unit: 'millisecond',\n      }).addCallback(this.reportUpstreamClockSkew);\n    }\n  }\n\n  readonly reportUpstreamLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.receiveTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportReplicaLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.receiveTimeMs);\n    }\n  };\n\n  readonly reportTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportLastTotalLag = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(last.replicateTimeMs - last.sendTimeMs);\n    }\n  };\n\n  readonly reportUpstreamClockSkew = (o: ObservableResult) => {\n    const last = this.#last?.lastTimings;\n    if (last) {\n      o.observe(estimateUpstreamClockSkewMs(last));\n    }\n  };\n}\n\n/**\n * Estimates how far the upstream database's clock is from ours, in\n * milliseconds, positive meaning upstream is ahead.\n *\n * `sendTimeMs` and `receiveTimeMs` are both stamped in the\n * replication-manager, either side of `commitTimeMs`, which upstream stamps.\n * Assuming the upstream commit fell midway through that round trip, the\n * difference between where it actually landed and that midpoint is the clock\n * offset -- the same round-trip estimate NTP uses.\n *\n * The estimate is biased by any asymmetry between the two legs: the outbound\n * leg is a SQL round trip, while the return leg additionally waits on a WAL\n * flush and the walsender (which the pre-PG-17 path notes can add 50~100ms on\n * an idle database). So treat this as a detector of gross skew -- the kind\n * that invalidates sync.e2e_serving_lag -- and not as a sub-100ms correction.\n *\n * The rigorous bound, if a tighter reading is ever needed: the commit\n * genuinely happened between send and receive, so a `commitTimeMs` outside\n * that window proves skew of at least the amount by which it falls outside,\n * independent of any latency assumption.\n */\nexport function estimateUpstreamClockSkewMs({\n  sendTimeMs,\n  commitTimeMs,\n  receiveTimeMs,\n}: {\n  sendTimeMs: number;\n  commitTimeMs: number;\n  receiveTimeMs: number;\n}): number {\n  return commitTimeMs - (sendTimeMs + receiveTimeMs) / 2;\n}\n"],"mappings":";;AAMA,IAAM,uCACJ,QAAQ,IAAI,8CAA8C;AAE5D,IAAa,4BAAb,MAAuC;CACrC;CACA,QAAkC;CAElC,YAAY,IAAgB;EAC1B,KAAKA,MAAM;CACb;CAEA,OAAO,QAA2B;EAChC,MAAM,QAAQ,KAAKC,UAAU;EAC7B,KAAKA,QAAQ;EAEb,MAAM,EAAC,gBAAe;EACtB,IAAI,aAAa;GACf,MAAM,QAAQ,YAAY,kBAAkB,YAAY;GACxD,IAAI,QAAQ,KACV,KAAKD,IAAI,OAAO,yBAAyB,MAAM,MAAM,MAAM;QACtD,IAAI,QAAQ,KACjB,KAAKA,IAAI,OAAO,oBAAoB,MAAM,MAAM,MAAM;GAExD,IAAI,sCACF,KAAKA,IAAI,QAAQ,mBAAmB,MAAM,MAAM,MAAM;EAE1D;EAEA,IAAI,OAAO;GACT,iBAAiB,eAAe,gBAAgB;IAC9C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,iBAAiB;GAErC,iBAAiB,eAAe,eAAe;IAC7C,aACE;IAEF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,gBAAgB;GAEpC,iBAAiB,eAAe,aAAa;IAC3C,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,cAAc;GAElC,iBAAiB,eAAe,kBAAkB;IAChD,aACE;IAIF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,kBAAkB;GAEtC,iBAAiB,eAAe,uBAAuB;IACrD,aACE;IASF,MAAM;GACR,CAAC,EAAE,YAAY,KAAK,uBAAuB;EAC7C;CACF;CAEA,qBAA8B,MAAwB;EACpD,MAAM,OAAO,KAAKC,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,gBAAgB,KAAK,UAAU;CAElD;CAEA,oBAA6B,MAAwB;EACnD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,aAAa;CAEvD;CAEA,kBAA2B,MAAwB;EACjD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,sBAA+B,MAAwB;EACrD,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,KAAK,kBAAkB,KAAK,UAAU;CAEpD;CAEA,2BAAoC,MAAwB;EAC1D,MAAM,OAAO,KAAKA,OAAO;EACzB,IAAI,MACF,EAAE,QAAQ,4BAA4B,IAAI,CAAC;CAE/C;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,SAAgB,4BAA4B,EAC1C,YACA,cACA,iBAKS;CACT,OAAO,gBAAgB,aAAa,iBAAiB;AACvD"}

package/out/zero-cache/src/services/replicator/reporter/report-schema.d.ts

@@ -21,8 +21,8 @@ export declare const replicationTimingsSchema: v.ObjectType<Omit<{
 }, undefined>;
 export declare const replicationReportSchema: v.ObjectType<{
     lastTimings: v.Optional<{
-        sendTimeMs: number;
         commitTimeMs: number;
+        sendTimeMs: number;
         receiveTimeMs: number;
         replicateTimeMs: number;
     }>;

package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.d.ts

@@ -0,0 +1,44 @@
+import type { ReplicaState } from '../replicator/replicator.ts';
+export type PendingUpstreamCommit = {
+    readonly watermark: string;
+    readonly commitTimeMs: number;
+};
+/**
+ * Pairs the upstream commit timestamps carried by `version-ready`
+ * notifications with the moment the corresponding change is poked to clients,
+ * yielding the end-to-end serving lag.
+ *
+ * This measures completion, not backlog: an observation is produced only when
+ * work actually reaches clients, so the resulting histogram is a latency
+ * distribution rather than a periodic snapshot of how far behind things are.
+ * A ViewSyncer that is stuck contributes nothing until it recovers, instead of
+ * re-reporting its age on every sample tick.
+ */
+export declare class E2EServingLagTracker {
+    #private;
+    get pending(): PendingUpstreamCommit | null;
+    /**
+     * Records the upstream commit behind a `version-ready` notification.
+     *
+     * Notifications coalesce when the ViewSyncer is busy, so one state may stand
+     * in for several commits. The *oldest* commit time is kept, since it bounds
+     * the lag of everything the notification subsumed, while the watermark
+     * advances to the newest — that is the one that must be served for all of
+     * the subsumed commits to have been delivered.
+     */
+    onVersionReady({ watermark, upstreamCommitTimeMs }: ReplicaState): void;
+    /**
+     * Called once a version has been poked to clients.
+     *
+     * @return the observation to record, or `null` if the served version does
+     *     not yet cover an outstanding upstream commit.
+     */
+    onVersionServed(servedVersion: string, nowMs: number): Observation | null;
+}
+export type Observation = {
+    /** End-to-end lag in milliseconds, clamped to be non-negative. */
+    readonly lagMs: number;
+    /** Whether the clamp was applied, i.e. the raw measurement was negative. */
+    readonly clamped: boolean;
+};
+//# sourceMappingURL=e2e-serving-lag.d.ts.map

package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"e2e-serving-lag.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/e2e-serving-lag.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,oBAAoB;;IAG/B,IAAI,OAAO,IAAI,qBAAqB,GAAG,IAAI,CAE1C;IAED;;;;;;;;OAQG;IACH,cAAc,CAAC,EAAC,SAAS,EAAE,oBAAoB,EAAC,EAAE,YAAY,GAAG,IAAI;IAcrE;;;;;OAKG;IACH,eAAe,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI;CAoB1E;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kEAAkE;IAClE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,4EAA4E;IAC5E,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B,CAAC"}

package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.js

@@ -0,0 +1,59 @@
+//#region ../zero-cache/src/services/view-syncer/e2e-serving-lag.ts
+/**
+* Pairs the upstream commit timestamps carried by `version-ready`
+* notifications with the moment the corresponding change is poked to clients,
+* yielding the end-to-end serving lag.
+*
+* This measures completion, not backlog: an observation is produced only when
+* work actually reaches clients, so the resulting histogram is a latency
+* distribution rather than a periodic snapshot of how far behind things are.
+* A ViewSyncer that is stuck contributes nothing until it recovers, instead of
+* re-reporting its age on every sample tick.
+*/
+var E2EServingLagTracker = class {
+	#pending = null;
+	get pending() {
+		return this.#pending;
+	}
+	/**
+	* Records the upstream commit behind a `version-ready` notification.
+	*
+	* Notifications coalesce when the ViewSyncer is busy, so one state may stand
+	* in for several commits. The *oldest* commit time is kept, since it bounds
+	* the lag of everything the notification subsumed, while the watermark
+	* advances to the newest — that is the one that must be served for all of
+	* the subsumed commits to have been delivered.
+	*/
+	onVersionReady({ watermark, upstreamCommitTimeMs }) {
+		if (watermark === void 0 || upstreamCommitTimeMs === void 0) return;
+		const pending = this.#pending;
+		this.#pending = {
+			watermark,
+			commitTimeMs: pending === null ? upstreamCommitTimeMs : Math.min(pending.commitTimeMs, upstreamCommitTimeMs)
+		};
+	}
+	/**
+	* Called once a version has been poked to clients.
+	*
+	* @return the observation to record, or `null` if the served version does
+	*     not yet cover an outstanding upstream commit.
+	*/
+	onVersionServed(servedVersion, nowMs) {
+		const pending = this.#pending;
+		if (pending === null || servedVersion < pending.watermark) return null;
+		this.#pending = null;
+		const lagMs = nowMs - pending.commitTimeMs;
+		if (lagMs >= 0) return {
+			lagMs,
+			clamped: false
+		};
+		return {
+			lagMs: 0,
+			clamped: true
+		};
+	}
+};
+//#endregion
+export { E2EServingLagTracker };
+
+//# sourceMappingURL=e2e-serving-lag.js.map

package/out/zero-cache/src/services/view-syncer/e2e-serving-lag.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"e2e-serving-lag.js","names":["#pending"],"sources":["../../../../../../zero-cache/src/services/view-syncer/e2e-serving-lag.ts"],"sourcesContent":["import type {ReplicaState} from '../replicator/replicator.ts';\n\nexport type PendingUpstreamCommit = {\n  readonly watermark: string;\n  readonly commitTimeMs: number;\n};\n\n/**\n * Pairs the upstream commit timestamps carried by `version-ready`\n * notifications with the moment the corresponding change is poked to clients,\n * yielding the end-to-end serving lag.\n *\n * This measures completion, not backlog: an observation is produced only when\n * work actually reaches clients, so the resulting histogram is a latency\n * distribution rather than a periodic snapshot of how far behind things are.\n * A ViewSyncer that is stuck contributes nothing until it recovers, instead of\n * re-reporting its age on every sample tick.\n */\nexport class E2EServingLagTracker {\n  #pending: PendingUpstreamCommit | null = null;\n\n  get pending(): PendingUpstreamCommit | null {\n    return this.#pending;\n  }\n\n  /**\n   * Records the upstream commit behind a `version-ready` notification.\n   *\n   * Notifications coalesce when the ViewSyncer is busy, so one state may stand\n   * in for several commits. The *oldest* commit time is kept, since it bounds\n   * the lag of everything the notification subsumed, while the watermark\n   * advances to the newest — that is the one that must be served for all of\n   * the subsumed commits to have been delivered.\n   */\n  onVersionReady({watermark, upstreamCommitTimeMs}: ReplicaState): void {\n    if (watermark === undefined || upstreamCommitTimeMs === undefined) {\n      return;\n    }\n    const pending = this.#pending;\n    this.#pending = {\n      watermark,\n      commitTimeMs:\n        pending === null\n          ? upstreamCommitTimeMs\n          : Math.min(pending.commitTimeMs, upstreamCommitTimeMs),\n    };\n  }\n\n  /**\n   * Called once a version has been poked to clients.\n   *\n   * @return the observation to record, or `null` if the served version does\n   *     not yet cover an outstanding upstream commit.\n   */\n  onVersionServed(servedVersion: string, nowMs: number): Observation | null {\n    const pending = this.#pending;\n    if (pending === null || servedVersion < pending.watermark) {\n      return null;\n    }\n    this.#pending = null;\n\n    const lagMs = nowMs - pending.commitTimeMs;\n    if (lagMs >= 0) {\n      return {lagMs, clamped: false};\n    }\n    // The commit time is on the upstream database's clock while `nowMs` is\n    // local, so a negative duration means upstream's clock is running ahead of\n    // ours by more than the entire pipeline latency. Clamp, because a negative\n    // value would corrupt the histogram's sum -- but report the clamp, since\n    // it is proof of gross clock skew, and skew in this direction biases the\n    // metric *low*, which reads as healthy serving rather than as a broken\n    // measurement. See replication.upstream_clock_skew for the magnitude.\n    return {lagMs: 0, clamped: true};\n  }\n}\n\nexport type Observation = {\n  /** End-to-end lag in milliseconds, clamped to be non-negative. */\n  readonly lagMs: number;\n  /** Whether the clamp was applied, i.e. the raw measurement was negative. */\n  readonly clamped: boolean;\n};\n"],"mappings":";;;;;;;;;;;;AAkBA,IAAa,uBAAb,MAAkC;CAChC,WAAyC;CAEzC,IAAI,UAAwC;EAC1C,OAAO,KAAKA;CACd;;;;;;;;;;CAWA,eAAe,EAAC,WAAW,wBAA2C;EACpE,IAAI,cAAc,KAAA,KAAa,yBAAyB,KAAA,GACtD;EAEF,MAAM,UAAU,KAAKA;EACrB,KAAKA,WAAW;GACd;GACA,cACE,YAAY,OACR,uBACA,KAAK,IAAI,QAAQ,cAAc,oBAAoB;EAC3D;CACF;;;;;;;CAQA,gBAAgB,eAAuB,OAAmC;EACxE,MAAM,UAAU,KAAKA;EACrB,IAAI,YAAY,QAAQ,gBAAgB,QAAQ,WAC9C,OAAO;EAET,KAAKA,WAAW;EAEhB,MAAM,QAAQ,QAAQ,QAAQ;EAC9B,IAAI,SAAS,GACX,OAAO;GAAC;GAAO,SAAS;EAAK;EAS/B,OAAO;GAAC,OAAO;GAAG,SAAS;EAAI;CACjC;AACF"}

package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"view-syncer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/view-syncer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,2BAA2B,EAAC,MAAM,yDAAyD,CAAC;AACzG,OAAO,KAAK,EAEV,qBAAqB,EACtB,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,iDAAiD,CAAC;AAC1F,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAQtE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAGpF,OAAO,EAAkB,KAAK,IAAI,EAAC,MAAM,oBAAoB,CAAC;AAK9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EACV,sBAAsB,EAEvB,MAAM,yCAAyC,CAAC;AAOjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAM1E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAQxD,OAAO,KAAK,EAEV,wBAAwB,EACxB,kBAAkB,EAEnB,MAAM,iCAAiC,CAAC;AAUzC,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAuCzD,MAAM,WAAW,UAAU;IACzB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC,CAAC;IAEtB,oBAAoB,CAClB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,aAAa,CACX,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAErB,OAAO,CAAC,QAAQ,EAAE,kBAAkB,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,UAAU,CACR,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,iBAAiB,EACtB,mBAAmB,EAAE,OAAO,GAC3B,OAAO,CAAC,IAAI,CAAC,CAAC;IAGjB,kBAAkB,EAAE,wBAAwB,CAAC;IAE7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,MAAM,WAAW,GAAG,kBAAkB,GAAG;IAC7C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC;CACjC,CAAC;AAmBF,KAAK,UAAU,GAAG,CAChB,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAChC,KAAK,CAAC,EAAE,MAAM,KACX,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAIvC,qBAAa,iBAAkB,YAAW,UAAU,EAAE,oBAAoB;;IACxE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,SAAc;IAGlC,QAAQ,CAAC,kBAAkB,EAAE,wBAAwB,CAAC;gBA+JpD,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,UAAU,EACjB,cAAc,EAAE,cAAc,EAC9B,cAAc,EAAE,YAAY,CAAC,YAAY,CAAC,EAC1C,gBAAgB,EAAE,gBAAgB,EAClC,oBAAoB,EAAE,MAAM,EAC5B,iBAAiB,EAAE,iBAAiB,EACpC,kBAAkB,EAAE,wBAAwB,EAC5C,sBAAsB,EAAE,sBAAsB,GAAG,SAAS,EAC1D,aAAa,EAAE,CAAC,CAAC,EACf,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,KACjB,OAAO,CAAC,CAAC,CAAC,EACf,WAAW,SAAuB,EAClC,YAAY,GAAE,UAAwC;IAiGxD,UAAU,IAAI,OAAO,CAAC,aAAa,GAAG,UAAU,CAAC;IAO3C,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAgI1B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAEjC;IAED,IAAI,kBAAkB,IAAI,OAAO,CAKhC;IAQD;;;;;;;;OAQG;IACH,SAAS,IAAI,OAAO;IAkKpB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC;IAuGf,oBAAoB,CACxB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC;IAiBV,UAAU,CACd,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,iBAAiB,EACtB,mBAAmB,EAAE,OAAO,GAC3B,OAAO,CAAC,IAAI,CAAC;IAsCV,aAAa,CACjB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC;IAikDd,OAAO,CACX,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,IAAI,CAAC;IAmKhB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAgCrB;;;OAGG;IACH,eAAe;CAGhB;AAsGD,qBAAa,cAAc;;gBAKb,EAAE,EAAE,UAAU;IAIpB,KAAK;IAOX,oBAAoB;IAMd,YAAY,CAAC,cAAc,CAAC,EAAE,MAAM;IAW1C,UAAU;IAWV,sCAAsC;IACtC,IAAI,IAAI,MAAM;IAKd;;;OAGG;IACH,YAAY,IAAI,MAAM;CAKvB"}
+{"version":3,"file":"view-syncer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/view-syncer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,2BAA2B,EAAC,MAAM,yDAAyD,CAAC;AACzG,OAAO,KAAK,EAEV,qBAAqB,EACtB,MAAM,0CAA0C,CAAC;AAElD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,iDAAiD,CAAC;AAC1F,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAQtE,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAGpF,OAAO,EAAkB,KAAK,IAAI,EAAC,MAAM,oBAAoB,CAAC;AAK9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EACV,sBAAsB,EAEvB,MAAM,yCAAyC,CAAC;AAOjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAM1E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAQxD,OAAO,KAAK,EAEV,wBAAwB,EACxB,kBAAkB,EAEnB,MAAM,iCAAiC,CAAC;AAUzC,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAG7D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAuCzD,MAAM,WAAW,UAAU;IACzB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC,CAAC;IAEtB,oBAAoB,CAClB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,aAAa,CACX,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAErB,OAAO,CAAC,QAAQ,EAAE,kBAAkB,EAAE,GAAG,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,UAAU,CACR,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,iBAAiB,EACtB,mBAAmB,EAAE,OAAO,GAC3B,OAAO,CAAC,IAAI,CAAC,CAAC;IAGjB,kBAAkB,EAAE,wBAAwB,CAAC;IAE7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;CACtC;AAED,MAAM,MAAM,WAAW,GAAG,kBAAkB,GAAG;IAC7C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC;CACjC,CAAC;AAmBF,KAAK,UAAU,GAAG,CAChB,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAChC,KAAK,CAAC,EAAE,MAAM,KACX,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAIvC,qBAAa,iBAAkB,YAAW,UAAU,EAAE,oBAAoB;;IACxE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,SAAc;IAGlC,QAAQ,CAAC,kBAAkB,EAAE,wBAAwB,CAAC;gBA0LpD,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,UAAU,EACjB,cAAc,EAAE,cAAc,EAC9B,cAAc,EAAE,YAAY,CAAC,YAAY,CAAC,EAC1C,gBAAgB,EAAE,gBAAgB,EAClC,oBAAoB,EAAE,MAAM,EAC5B,iBAAiB,EAAE,iBAAiB,EACpC,kBAAkB,EAAE,wBAAwB,EAC5C,sBAAsB,EAAE,sBAAsB,GAAG,SAAS,EAC1D,aAAa,EAAE,CAAC,CAAC,EACf,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,KACjB,OAAO,CAAC,CAAC,CAAC,EACf,WAAW,SAAuB,EAClC,YAAY,GAAE,UAAwC;IAiGxD,UAAU,IAAI,OAAO,CAAC,aAAa,GAAG,UAAU,CAAC;IAO3C,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAkI1B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAEjC;IAED,IAAI,kBAAkB,IAAI,OAAO,CAKhC;IAkBD;;;;;;;;OAQG;IACH,SAAS,IAAI,OAAO;IAkKpB,cAAc,CACZ,QAAQ,EAAE,kBAAkB,EAC5B,qBAAqB,EAAE,qBAAqB,GAC3C,MAAM,CAAC,UAAU,CAAC;IAuGf,oBAAoB,CACxB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,2BAA2B,GAC/B,OAAO,CAAC,IAAI,CAAC;IAiBV,UAAU,CACd,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,iBAAiB,EACtB,mBAAmB,EAAE,OAAO,GAC3B,OAAO,CAAC,IAAI,CAAC;IAsCV,aAAa,CACjB,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,MAAM,EAAE,CAAC;IAikDd,OAAO,CACX,QAAQ,EAAE,kBAAkB,EAC5B,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,IAAI,CAAC;IAmKhB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAgCrB;;;OAGG;IACH,eAAe;CAGhB;AAsGD,qBAAa,cAAc;;gBAKb,EAAE,EAAE,UAAU;IAIpB,KAAK;IAOX,oBAAoB;IAMd,YAAY,CAAC,cAAc,CAAC,EAAE,MAAM;IAW1C,UAAU;IAWV,sCAAsC;IACtC,IAAI,IAAI,MAAM;IAKd;;;OAGG;IACH,YAAY,IAAI,MAAM;CAKvB"}

package/out/zero-cache/src/services/view-syncer/view-syncer.js

@@ -25,6 +25,7 @@ import { ClientHandler, startPoke } from "./client-handler.js";
 import { tracer } from "./tracer.js";
 import { CVRStore, ClientNotFoundError } from "./cvr-store.js";
 import { CVRConfigDrivenUpdater, CVRQueryDrivenUpdater, nextEvictionTime } from "./cvr.js";
+import { E2EServingLagTracker } from "./e2e-serving-lag.js";
 import { handleInspect } from "./inspect-handler.js";
 import { QueryCoveringIndex } from "./query-covering.js";
 import { resolver } from "@rocicorp/resolver";
@@ -90,6 +91,7 @@ var ViewSyncerService = class {
 	#cvr;
 	#pipelinesSynced = false;
 	#servedVersion = null;
+	#e2eServingLagTracker = new E2EServingLagTracker();
 	#expiredQueriesTimer = 0;
 	#authMaintenanceTimer = 0;
 	#setTimeout;
@@ -102,6 +104,14 @@ var ViewSyncerService = class {
 		description: "Time from ViewSyncer query sync requiring hydration to output for a client group. Includes query transformation, query materialization, CVR flush, catchup, and pokeEnd.",
 		unit: "s"
 	});
+	#e2eServingLag = getOrCreateNativeHistogram("sync", "e2e_serving_lag", {
+		description: "End-to-end lag from upstream commit to ViewSyncer output. Spans the whole pipeline: the upstream transaction commit, replication to the replica, IVM advancement, CVR flush, and pokeEnd. Recorded once per served version, not sampled, so each observation is the completion latency of real replicated work.",
+		unit: "s"
+	});
+	#e2eServingLagClamps = getOrCreateCounter("sync", "e2e_serving_lag_clamps", {
+		description: "Observations of sync.e2e_serving_lag that came out negative and were clamped to zero. Non-zero means the upstream database clock is running ahead of this pod by more than the entire pipeline latency, so sync.e2e_serving_lag is biased low and reads healthier than reality. See replication.upstream_clock_skew for the magnitude.",
+		unit: "{observation}"
+	});
 	#transactionAdvanceTime = getOrCreateLatencyHistogram("sync", "advance-time", "Time to advance all queries for a given client group after applying a new transaction to the replica.");
 	#queryTransformations = getOrCreateCounter("sync", "query.transformations", "Number of query transformations performed");
 	#queryTransformationTime = getOrCreateLatencyHistogram("sync", "query.transformation-time", "Time to transform custom queries via API server.");
@@ -186,12 +196,14 @@ var ViewSyncerService = class {
 				this.#lc.debug?.(`draining view-syncer ${this.id} before running`);
 				this.stop();
 			}
-			for await (const { state } of this.#stateChanges) {
+			for await (const replicaState of this.#stateChanges) {
+				const { state } = replicaState;
 				if (this.#drainCoordinator.shouldDrain()) {
 					this.#lc.debug?.(`draining view-syncer ${this.id} (elective)`);
 					break;
 				}
 				assert(state === "version-ready", "state should be version-ready");
+				this.#e2eServingLagTracker.onVersionReady(replicaState);
 				await this.#runInLockWithCVR(async (lc, cvr) => {
 					const clientSchema = must(cvr.clientSchema, "cvr.clientSchema missing after initialization");
 					if (!this.#pipelines.initialized()) this.#pipelines.init(clientSchema);
@@ -258,6 +270,11 @@ var ViewSyncerService = class {
 	}
 	#markVersionServed(version) {
 		this.#servedVersion = version.stateVersion;
+		const observation = this.#e2eServingLagTracker.onVersionServed(version.stateVersion, Date.now());
+		if (observation !== null) {
+			this.#e2eServingLag.recordMs(observation.lagMs);
+			if (observation.clamped) this.#e2eServingLagClamps.add(1);
+		}
 	}
 	#keepAliveUntil = 0;
 	/**