@rocicorp/zero 1.9.1-canary.1 → 1.9.1-canary.2

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts

@@ -3,12 +3,24 @@ import type { WatermarkedChange } from './change-streamer-service.ts';
 import type { Status } from './change-streamer.ts';
 import type { Subscriber } from './subscriber.ts';
 export type ProgressMonitorOptions = {
-    flowControlConsensusPaddingSeconds: number;
+    flowControlConsensusTimeoutProportion: number;
+    flowControlSlowSubscriberGracePeriodMs?: number | undefined;
+    setTimeoutFn?: typeof setTimeout | undefined;
+    clearTimeoutFn?: typeof clearTimeout | undefined;
 };
 export declare class Forwarder {
     #private;
     constructor(lc: LogContext, opts?: ProgressMonitorOptions);
     startProgressMonitor(): void;
+    /**
+     * Runs one progress-monitor tick: samples each active subscriber's change
+     * rate, logs slow broadcasts, and disconnects subscribers that have been
+     * continuously lagging for longer than the configured grace period.
+     *
+     * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather
+     * than private) so tests can drive a single tick with an explicit `now`.
+     */
+    checkSubscriberProgress(now: number): void;
     stopProgressMonitor(): void;
     /**
      * `add()` is called in lock step with `Storer.catchup()` so that the

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAY,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AAC/E,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,sBAAsB,CAAC;AACjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,sBAAsB,GAAG;IACnC,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF,qBAAa,SAAS;;gBAqBlB,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAAgE;IAkDxE,oBAAoB;IAsBpB,mBAAmB;IAInB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IAKhC,UAAU,CAAC,MAAM,EAAE,MAAM;IAMzB;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB;IAiErD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}
+{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAY,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AAC/E,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,sBAAsB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAkB,MAAM,iBAAiB,CAAC;AAEjE,MAAM,MAAM,sBAAsB,GAAG;IACnC,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5D,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IAC7C,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF,qBAAa,SAAS;;gBAyBlB,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAEL;IA+DH,oBAAoB;IAQpB;;;;;;;OAOG;IACH,uBAAuB,CAAC,GAAG,EAAE,MAAM;IAyCnC,mBAAmB;IAInB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IAKhC,UAAU,CAAC,MAAM,EAAE,MAAM;IAMzB;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB;IAsErD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}

package/out/zero-cache/src/services/change-streamer/forwarder.js

@@ -4,17 +4,27 @@ import { Broadcast } from "./broadcast.js";
 //#region ../zero-cache/src/services/change-streamer/forwarder.ts
 var Forwarder = class {
 	#lc;
-	#progressMonitorOptions;
+	#setTimeout;
+	#clearTimeout;
 	#active = /* @__PURE__ */ new Set();
 	#queued = /* @__PURE__ */ new Set();
+	#earlyReleaseOptions;
+	#flowControlSlowSubscriberGracePeriodMs;
 	#flowControlWaits = getOrCreateCounter("replication", "flow_control.waits", "Flow-control checkpoints completed, labeled by release mode.");
 	#flowControlWaitTime = getOrCreateLatencyHistogram("replication", "flow_control.wait_duration", "Time replication waits at flow-control checkpoints.");
 	#inTransaction = false;
 	#currentBroadcast;
 	#progressMonitor;
-	constructor(lc, opts = { flowControlConsensusPaddingSeconds: 1 }) {
-		this.#lc = lc.withContext("component", "progress-monitor");
-		this.#progressMonitorOptions = opts;
+	constructor(lc, opts = { flowControlConsensusTimeoutProportion: 2 }) {
+		this.#lc = lc.withContext("component", "subscriber-progress-monitor");
+		this.#setTimeout = opts.setTimeoutFn ?? setTimeout;
+		this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;
+		this.#earlyReleaseOptions = opts.flowControlConsensusTimeoutProportion < 0 ? void 0 : {
+			consensusTimeoutProportion: opts.flowControlConsensusTimeoutProportion,
+			setTimeoutFn: this.#setTimeout,
+			clearTimeoutFn: this.#clearTimeout
+		};
+		this.#flowControlSlowSubscriberGracePeriodMs = opts.flowControlSlowSubscriberGracePeriodMs;
 		getOrCreateGauge("replication", "flow_control.active_subscribers", "Active change-stream subscribers receiving live changes.").addCallback((result) => result.observe(this.#active.size));
 		getOrCreateGauge("replication", "flow_control.queued_subscribers", "Change-stream subscribers waiting for the current transaction to finish before activation.").addCallback((result) => result.observe(this.#queued.size));
 		getOrCreateGauge("replication", "flow_control.pending_messages", "Downstream change-stream messages not yet acked by subscribers.").addCallback((result) => result.observe(this.#subscriberStats().pendingMessages));
@@ -30,14 +40,34 @@ var Forwarder = class {
 	}
 	startProgressMonitor() {
 		clearInterval(this.#progressMonitor);
-		this.#progressMonitor = setInterval(this.#trackProgress, 1e3);
+		this.#progressMonitor = setInterval(() => this.checkSubscriberProgress(performance.now()), 1e3);
+	}
+	/**
+	* Runs one progress-monitor tick: samples each active subscriber's change
+	* rate, logs slow broadcasts, and disconnects subscribers that have been
+	* continuously lagging for longer than the configured grace period.
+	*
+	* Invoked on an interval by {@link startProgressMonitor()}; exposed (rather
+	* than private) so tests can drive a single tick with an explicit `now`.
+	*/
+	checkSubscriberProgress(now) {
+		let timedOut;
+		let slowestOnTime;
+		for (const sub of this.#active) {
+			const stats = sub.sampleProcessRate(now).getStats();
+			if (stats.missedLastTimeout) (timedOut ??= /* @__PURE__ */ new Map()).set(sub, stats);
+			else if (stats.processRate <= (slowestOnTime?.processRate ?? Infinity)) slowestOnTime = stats;
+		}
+		this.#currentBroadcast?.logSlowRequests(now);
+		if (this.#flowControlSlowSubscriberGracePeriodMs && timedOut?.size && slowestOnTime) for (const [sub, stats] of timedOut.entries()) {
+			const rate = stats.processRate < slowestOnTime.processRate ? "lagging" : "catching-up";
+			const laggingDuration = sub.reportChangeRate(now, rate);
+			if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {
+				this.#lc.warn?.(`subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`, { stats });
+				sub.close(3, `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`);
+			}
+		}
 	}
-	#trackProgress = () => {
-		const now = performance.now();
-		for (const sub of this.#active) sub.sampleProcessRate(now);
-		const { flowControlConsensusPaddingSeconds } = this.#progressMonitorOptions;
-		if (flowControlConsensusPaddingSeconds >= 0) this.#currentBroadcast?.checkProgress(this.#lc, flowControlConsensusPaddingSeconds * 1e3, now);
-	};
 	stopProgressMonitor() {
 		clearInterval(this.#progressMonitor);
 	}
@@ -77,7 +107,7 @@ var Forwarder = class {
 	*/
 	async forwardWithFlowControl(entry) {
 		const start = performance.now();
-		const broadcast = new Broadcast(this.#active.values(), entry);
+		const broadcast = new Broadcast(this.#lc, this.#active.values(), entry, this.#earlyReleaseOptions);
 		this.#updateActiveSubscribers(entry[1]);
 		this.#currentBroadcast = broadcast;
 		try {

package/out/zero-cache/src/services/change-streamer/forwarder.js.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.js","names":["#lc","#progressMonitorOptions","#active","#queued","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#trackProgress","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast} from './broadcast.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer-service.ts';\nimport type {Status} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusPaddingSeconds: number;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #progressMonitorOptions: ProgressMonitorOptions;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {flowControlConsensusPaddingSeconds: 1},\n  ) {\n    this.#lc = lc.withContext('component', 'progress-monitor');\n    this.#progressMonitorOptions = opts;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(this.#trackProgress, 1000);\n  }\n\n  readonly #trackProgress = () => {\n    const now = performance.now();\n    for (const sub of this.#active) {\n      sub.sampleProcessRate(now);\n    }\n\n    const {flowControlConsensusPaddingSeconds} = this.#progressMonitorOptions;\n    // A negative number disables early flow control release.\n    if (flowControlConsensusPaddingSeconds >= 0) {\n      this.#currentBroadcast?.checkProgress(\n        this.#lc,\n        flowControlConsensusPaddingSeconds * 1000,\n        now,\n      );\n    }\n  };\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const start = performance.now();\n    const broadcast = new Broadcast(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAgBA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAAC,oCAAoC,EAAC,GACrE;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKC,0BAA0B;EAE/B,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKG,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,YAAY,KAAKC,gBAAgB,GAAI;CAC/D;CAEA,uBAAgC;EAC9B,MAAM,MAAM,YAAY,IAAI;EAC5B,KAAK,MAAM,OAAO,KAAKN,SACrB,IAAI,kBAAkB,GAAG;EAG3B,MAAM,EAAC,uCAAsC,KAAKD;EAElD,IAAI,sCAAsC,GACxC,KAAKQ,mBAAmB,cACtB,KAAKT,KACL,qCAAqC,KACrC,GACF;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKO,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKG,gBACP,KAAKP,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKS,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKT,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UAAU,KAAKA,QAAQ,OAAO,GAAG,KAAK;EAC5D,KAAKS,yBAAyB,MAAM,EAAE;EAGtC,KAAKF,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKL,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKI,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKP,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKO,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKP,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}
+{"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer-service.ts';\nimport type {Status} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusTimeoutProportion: number;\n  flowControlSlowSubscriberGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #earlyReleaseOptions: EarlyReleaseOptions | undefined;\n  readonly #flowControlSlowSubscriberGracePeriodMs: number | undefined;\n\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {\n      flowControlConsensusTimeoutProportion: 2.0,\n    },\n  ) {\n    this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    this.#earlyReleaseOptions =\n      opts.flowControlConsensusTimeoutProportion < 0\n        ? undefined\n        : {\n            consensusTimeoutProportion:\n              opts.flowControlConsensusTimeoutProportion,\n            setTimeoutFn: this.#setTimeout,\n            clearTimeoutFn: this.#clearTimeout,\n          };\n    this.#flowControlSlowSubscriberGracePeriodMs =\n      opts.flowControlSlowSubscriberGracePeriodMs;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(\n      () => this.checkSubscriberProgress(performance.now()),\n      1000,\n    );\n  }\n\n  /**\n   * Runs one progress-monitor tick: samples each active subscriber's change\n   * rate, logs slow broadcasts, and disconnects subscribers that have been\n   * continuously lagging for longer than the configured grace period.\n   *\n   * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n   * than private) so tests can drive a single tick with an explicit `now`.\n   */\n  checkSubscriberProgress(now: number) {\n    let timedOut: Map<Subscriber, SubscriberStats> | undefined;\n    let slowestOnTime: SubscriberStats | undefined;\n\n    for (const sub of this.#active) {\n      const stats = sub.sampleProcessRate(now).getStats();\n      if (stats.missedLastTimeout) {\n        (timedOut ??= new Map()).set(sub, stats);\n      } else if (\n        stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n      ) {\n        slowestOnTime = stats;\n      }\n    }\n    this.#currentBroadcast?.logSlowRequests(now);\n\n    if (\n      this.#flowControlSlowSubscriberGracePeriodMs &&\n      timedOut?.size &&\n      slowestOnTime\n    ) {\n      for (const [sub, stats] of timedOut.entries()) {\n        const rate =\n          stats.processRate < slowestOnTime.processRate\n            ? 'lagging'\n            : 'catching-up';\n        const laggingDuration = sub.reportChangeRate(now, rate);\n        if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n          this.#lc.warn?.(\n            `subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n            {stats},\n          );\n          sub.close(\n            StreamTooFarBehind,\n            `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n          );\n        }\n      }\n    }\n  }\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const start = performance.now();\n    const broadcast = new Broadcast(\n      this.#lc,\n      this.#active.values(),\n      entry,\n      this.#earlyReleaseOptions,\n    );\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAqBA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAAyC;IACnE,KAAKN,IAAI,OACP,cAAc,IAAI,GAAG,kBAAkB,gBAAgB,oBACvD,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EACF;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKE,gBACP,KAAKR,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKU,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKV,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKH,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKQ,yBAAyB,MAAM,EAAE;EAGtC,KAAKF,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKR,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKQ,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKR,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -1,4 +1,5 @@
 import type { LogContext } from '@rocicorp/logger';
+import { type JSONObject } from '../../../../shared/src/bigint-json.ts';
 import { TransactionPool } from '../../db/transaction-pool.ts';
 import { type PostgresDB } from '../../types/pg.ts';
 import { type ShardID } from '../../types/shards.ts';
@@ -100,4 +101,37 @@ export declare class PurgeLocker {
     constructor(lc: LogContext, shard: ShardID, db: PostgresDB);
     acquire(): Promise<PurgeLock | null>;
 }
+type TaskKey = JSONObject;
+type TaskDoneFn = () => void;
+/**
+ * Periodic monitor for detecting the pathological condition in which a db
+ * query hangs forever because of a connection problem, such as a half-closed
+ * socket.
+ *
+ * Use this for db calls that recur or accumulate progress within a single
+ * logical operation on the main storer loop (e.g. the queue-processing loop,
+ * or a multi-batch catchup read), where "no progress for N ms" is the
+ * meaningful signal. For a one-off db call -- including ones made before
+ * {@link Storer.run} starts this monitor's polling -- use a plain timeout
+ * (see `Storer.#withTimeout`) instead.
+ *
+ * The failure handler should stop the server. The ProgressMonitor will
+ * assume this and stop itself after calling it.
+ *
+ * Internal to the Storer, but exported for testing.
+ */
+export declare class ProgressMonitor {
+    #private;
+    constructor(lc: LogContext, failureThresholdMs: number, onFailure: (err: Error) => void);
+    start(): void;
+    checkTaskProgress(now: Date): void;
+    /**
+     * Starts a task to be tracked.
+     *
+     * @return a callback to invoke when the task is done.
+     */
+    trackTask(task: TaskKey, now?: Date): TaskDoneFn;
+    stop(): void;
+}
+export {};
 //# sourceMappingURL=storer.d.ts.map

package/out/zero-cache/src/services/change-streamer/storer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,MAAM,EAKZ,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EACV,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAS3C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAuChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAoBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACvD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa;IAiCZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA2B5C,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;KACrC,CAAC;IAkCI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAOzD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,MAAM,CAAC,CAAC,EAAE,uBAAuB;IAIjC,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAOpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IA6FzC;;;OAGG;IACG,GAAG;IA2eT;;;OAGG;IACG,YAAY;IAQlB;;;;OAIG;IACG,IAAI;CAaX;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,UAmB/B;AA+BD,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd"}
+{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,uCAAuC,CAAC;AAM/C,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,MAAM,EAKZ,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EACV,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAS3C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAuChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAqBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACvD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa;IA+DZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA8B5C,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;KACrC,CAAC;IAqCI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAazD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,MAAM,CAAC,CAAC,EAAE,uBAAuB;IAIjC,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAMpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IA2EzC;;;OAGG;IACG,GAAG;IAshBT;;;OAGG;IACG,YAAY;IAQlB;;;;OAIG;IACG,IAAI;CAcX;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,UAmB/B;AA+BD,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd;AAKD,KAAK,OAAO,GAAG,UAAU,CAAC;AAE1B,KAAK,UAAU,GAAG,MAAM,IAAI,CAAC;AAE7B;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,eAAe;;gBASxB,EAAE,EAAE,UAAU,EACd,kBAAkB,EAAE,MAAM,EAC1B,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI;IAOjC,KAAK;IASL,iBAAiB,CAAC,GAAG,EAAE,IAAI;IAkB3B;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,OAAa,GAAG,UAAU;IAOtD,IAAI;CAIL"}

package/out/zero-cache/src/services/change-streamer/storer.js

@@ -66,6 +66,7 @@ var Storer = class {
 	#statementTimeoutMs;
 	#changeLogBatchSize;
 	#drainTimeoutMs;
+	#progressMonitor;
 	#approximateQueuedBytes = 0;
 	#running = false;
 	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onConsumed, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
@@ -81,6 +82,7 @@ var Storer = class {
 		this.#statementTimeoutMs = statementTimeoutMs;
 		this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);
 		this.#drainTimeoutMs = drainTimeoutMs;
+		this.#progressMonitor = new ProgressMonitor(lc, statementTimeoutMs, onFatal);
 		const heapStats = getHeapStatistics();
 		this.#backPressureThresholdBytes = (heapStats.heap_size_limit - heapStats.used_heap_size) * backPressureLimitHeapProportion;
 		this.#lc.info?.(`Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of --max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) to absorb upstream spikes`, { heapStats });
@@ -88,6 +90,28 @@ var Storer = class {
 	#cdc(table) {
 		return this.#db(`${cdcSchema(this.#shard)}.${table}`);
 	}
+	/**
+	* Bounds a one-off db call (i.e. not part of the main storer loop or the
+	* background catchup read, which are tracked by the ProgressMonitor
+	* instead) with a plain timeout. This covers calls like
+	* {@link assumeOwnership} and {@link getStartStreamInitializationParameters}
+	* that are made by the caller *before* {@link run()} -- and thus before the
+	* ProgressMonitor's polling starts -- as well as ones made well after,
+	* where a continuously-polling watchdog would be overkill for a single
+	* bounded db round trip.
+	*
+	* Does not cancel `promise` on timeout (there is no cancellation mechanism
+	* for a postgres.js query); it just stops waiting for it, and swallows a
+	* later rejection so it doesn't surface as an unhandled rejection.
+	*/
+	async #withTimeout(name, promise) {
+		const result = await orTimeout(promise, this.#statementTimeoutMs);
+		if (result === "timed-out") {
+			promise.catch(() => {});
+			throw new AbortError(`${name} did not complete within ${this.#statementTimeoutMs}ms`);
+		}
+		return result;
+	}
 	async assumeOwnership(purgeLock) {
 		const db = this.#db;
 		const owner = this.#taskID;
@@ -96,16 +120,16 @@ var Storer = class {
 		const addressWithProtocol = ownerProtocol === "ws" ? ownerAddress : `${ownerProtocol}://${ownerAddress}`;
 		this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);
 		const start = performance.now();
-		await db`UPDATE ${this.#cdc("replicationState")} SET ${db({
+		await this.#withTimeout("assume-ownership", db`UPDATE ${this.#cdc("replicationState")} SET ${db({
 			owner,
 			ownerAddress: addressWithProtocol
-		})}`;
+		})}`);
 		const elapsed = (performance.now() - start).toFixed(2);
 		this.#lc.info?.(`assumed ownership at ${addressWithProtocol} (${elapsed} ms)`);
 		if (purgeLock) purgeLock.release();
 	}
 	async getStartStreamInitializationParameters() {
-		const [[{ lastWatermark }], result] = await runTx(this.#db, (sql) => [sql`
+		const [[{ lastWatermark }], result] = await this.#withTimeout("get-stream-params", runTx(this.#db, (sql) => [sql`
         SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}`, sql`
         SELECT 
             json_build_object(
@@ -119,15 +143,15 @@ var Storer = class {
           LEFT JOIN ${this.#cdc("tableMetadata")} as t
           ON (b."schema" = t."schema" AND b."table" = t."table")
           GROUP BY b."schema", b."table", t."metadata"
-        `], { mode: READONLY });
+        `], { mode: READONLY }));
 		return {
 			lastWatermark,
 			backfillRequests: parse(result, backfillRequestsSchema)
 		};
 	}
 	async getMinWatermarkForCatchup() {
-		const [{ minWatermark }] = await this.#db`
-      SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`;
+		const [{ minWatermark }] = await this.#withTimeout("get-min-watermark", this.#db`
+      SELECT min(watermark) as "minWatermark" FROM ${this.#cdc("changeLog")}`);
 		return minWatermark;
 	}
 	purgeRecordsBefore(watermark) {
@@ -186,30 +210,22 @@ var Storer = class {
 		}]);
 	}
 	#readyForMore = null;
-	#backpressureTimeout;
 	readyForMore() {
 		if (!this.#running) return;
 		if (this.#readyForMore === null && this.#approximateQueuedBytes > this.#backPressureThresholdBytes) {
 			this.#lc.warn?.(`applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\n\nTo inspect changeLog backlog in your change DB:\n  SELECT\n    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\n    change->>'tag' AS operation,\n    COUNT(*) AS count\n  FROM "<app_id>/cdc"."changeLog"\n  GROUP BY 1, 2\n  ORDER BY 3 DESC\n  LIMIT 20;`);
 			this.#readyForMore = resolver();
-			this.#resetBackpressureTimeout(true);
 		}
 		return this.#readyForMore?.promise;
 	}
-	#resetBackpressureTimeout(reschedule) {
-		clearTimeout(this.#backpressureTimeout);
-		this.#backpressureTimeout = reschedule ? setTimeout(() => {
-			this.#readyForMore?.reject(new AbortError(`No statement processed within the last ${this.#statementTimeoutMs}ms. Considering the change-log to be wedged.`));
-			this.#readyForMore = null;
-		}, this.#statementTimeoutMs) : void 0;
-	}
 	#maybeReleaseBackPressure() {
-		if (this.#readyForMore !== null) if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
-			this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
-			this.#readyForMore.resolve();
-			this.#readyForMore = null;
-			this.#resetBackpressureTimeout(false);
-		} else this.#resetBackpressureTimeout(true);
+		if (this.#readyForMore !== null) {
+			if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
+				this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
+				this.#readyForMore.resolve();
+				this.#readyForMore = null;
+			}
+		}
 	}
 	/**
 	* Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.
@@ -248,6 +264,7 @@ var Storer = class {
 		this.#running = true;
 		this.#stopped = stopped;
 		this.#lc.info?.("starting storer");
+		this.#progressMonitor.start();
 		let err;
 		try {
 			await this.#processQueue();
@@ -259,7 +276,6 @@ var Storer = class {
 				this.#readyForMore.resolve();
 				this.#readyForMore = null;
 			}
-			this.#resetBackpressureTimeout(false);
 			this.#cancelQueueEntries(this.#queue.drain().filter((entry) => entry !== void 0), err);
 			this.#running = false;
 			signalStopped();
@@ -285,9 +301,20 @@ var Storer = class {
 	async #processQueue() {
 		let tx = null;
 		let msg;
+		let lastTaskDone;
+		const nextMessage = async () => {
+			lastTaskDone?.();
+			lastTaskDone = void 0;
+			const entry = await this.#queue.dequeue();
+			if (entry !== "stop") lastTaskDone = this.#progressMonitor.trackTask({
+				task: "queue-entry",
+				type: entry[0]
+			});
+			return entry;
+		};
 		const catchupQueue = [];
 		try {
-			while ((msg = await this.#queue.dequeue()) !== "stop") {
+			while ((msg = await nextMessage()) !== "stop") {
 				const [msgType] = msg;
 				switch (msgType) {
 					case "ready": {
@@ -320,10 +347,7 @@ var Storer = class {
 					const { promise, resolve, reject } = resolver();
 					promise.catch(() => {});
 					tx = {
-						pool: new TransactionPool(this.#lc.withContext("watermark", watermark), {
-							mode: READ_COMMITTED,
-							statementResponseTimeout: this.#statementTimeoutMs
-						}),
+						pool: new TransactionPool(this.#lc.withContext("watermark", watermark), { mode: READ_COMMITTED }),
 						preCommitWatermark: watermark,
 						pos: 0,
 						startingReplicationState: promise,
@@ -394,6 +418,10 @@ var Storer = class {
 		const reader = new TransactionPool(this.#lc.withContext("pool", "catchup"), { mode: READONLY });
 		reader.run(this.#db);
 		let lastWatermark;
+		const catchupSnapshotted = this.#progressMonitor.trackTask({
+			task: "capture-catchup-snapshot",
+			subscribers: subs.map(({ subscriber: s }) => s.id)
+		});
 		try {
 			[{lastWatermark}] = await reader.processReadTask((sql) => sql`
         SELECT "lastWatermark" FROM ${this.#cdc("replicationState")}
@@ -401,10 +429,18 @@ var Storer = class {
 		} catch (e) {
 			subs.map(({ subscriber }) => subscriber.fail(e));
 			throw e;
+		} finally {
+			catchupSnapshotted();
 		}
 		Promise.all(subs.map((sub) => this.#catchup(sub, lastWatermark, reader))).finally(() => reader.setDone());
 	}
 	async #catchup({ subscriber: sub, mode }, lastWatermark, reader) {
+		let entriesReceived = 0;
+		let catchupProgressed = this.#progressMonitor.trackTask({
+			task: "catchup",
+			subscriber: sub.id,
+			entriesReceived
+		});
 		try {
 			await reader.processReadTask(async (tx) => {
 				const start = Date.now();
@@ -416,21 +452,32 @@ var Storer = class {
            WHERE watermark >= ${sub.watermark}
              AND watermark <= ${lastWatermark}
            ORDER BY watermark, pos`.cursor(2e3)) {
-					const start = performance.now();
-					await lastBatchConsumed;
-					const elapsed = performance.now() - start;
-					if (lastBatchConsumed) this.#lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
-					for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
-					else if (watermarkFound) {
-						lastBatchConsumed = sub.catchup(toDownstream(entry));
-						count++;
-					} else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
-					else {
-						this.#lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
-						sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
-						return;
+					catchupProgressed();
+					entriesReceived += entries.length;
+					try {
+						const start = performance.now();
+						await lastBatchConsumed;
+						const elapsed = performance.now() - start;
+						if (lastBatchConsumed) this.#lc[elapsed > 100 ? "info" : "debug"]?.(`waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`);
+						for (const entry of entries) if (entry.watermark === sub.watermark) watermarkFound = true;
+						else if (watermarkFound) {
+							lastBatchConsumed = sub.catchup(toDownstream(entry));
+							count++;
+						} else if (mode === "backup") throw new AutoResetSignal(`backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`);
+						else {
+							this.#lc.warn?.(`rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`);
+							sub.close(2, `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`);
+							return;
+						}
+					} finally {
+						catchupProgressed = this.#progressMonitor.trackTask({
+							task: "catchup",
+							subscriber: sub.id,
+							entriesReceived
+						});
 					}
 				}
+				catchupProgressed();
 				if (watermarkFound) {
 					await lastBatchConsumed;
 					this.#lc.info?.(`caught up ${sub.id} with ${count} changes (${Date.now() - start} ms)`);
@@ -444,6 +491,8 @@ var Storer = class {
 				this.#onFatal(err);
 			}
 			sub.fail(err);
+		} finally {
+			catchupProgressed();
 		}
 	}
 	/**
@@ -554,6 +603,7 @@ var Storer = class {
 	*/
 	async stop() {
 		if (this.#running) {
+			this.#progressMonitor.stop();
 			this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);
 			this.#queue.enqueue("stop");
 		}
@@ -653,6 +703,65 @@ var PurgeLocker = class {
 		return new PurgeLock(this.#lc, tx, replicaVersion, watermark);
 	}
 };
+/**
+* Periodic monitor for detecting the pathological condition in which a db
+* query hangs forever because of a connection problem, such as a half-closed
+* socket.
+*
+* Use this for db calls that recur or accumulate progress within a single
+* logical operation on the main storer loop (e.g. the queue-processing loop,
+* or a multi-batch catchup read), where "no progress for N ms" is the
+* meaningful signal. For a one-off db call -- including ones made before
+* {@link Storer.run} starts this monitor's polling -- use a plain timeout
+* (see `Storer.#withTimeout`) instead.
+*
+* The failure handler should stop the server. The ProgressMonitor will
+* assume this and stop itself after calling it.
+*
+* Internal to the Storer, but exported for testing.
+*/
+var ProgressMonitor = class {
+	#lc;
+	#failureThresholdMs;
+	#onFailure;
+	#taskUpdates = /* @__PURE__ */ new Map();
+	#timer;
+	constructor(lc, failureThresholdMs, onFailure) {
+		this.#lc = lc.withContext("component", "storer-progress-monitor");
+		this.#failureThresholdMs = failureThresholdMs;
+		this.#onFailure = onFailure;
+	}
+	start() {
+		clearInterval(this.#timer);
+		this.#timer = setInterval(() => this.checkTaskProgress(/* @__PURE__ */ new Date()), this.#failureThresholdMs).unref();
+	}
+	checkTaskProgress(now) {
+		for (const [task, lastUpdate] of this.#taskUpdates.entries()) if (now.getTime() - lastUpdate.getTime() >= this.#failureThresholdMs) {
+			this.#lc.error?.(`Last task update was more than ${this.#failureThresholdMs}ms ago`, {
+				task,
+				lastUpdate
+			});
+			this.#onFailure(/* @__PURE__ */ new Error(`Task failed to progress for over ${this.#failureThresholdMs}ms: ${BigIntJSON.stringify(task)}`));
+			this.stop();
+			return;
+		}
+	}
+	/**
+	* Starts a task to be tracked.
+	*
+	* @return a callback to invoke when the task is done.
+	*/
+	trackTask(task, now = /* @__PURE__ */ new Date()) {
+		this.#taskUpdates.set(task, now);
+		return () => {
+			this.#taskUpdates.delete(task);
+		};
+	}
+	stop() {
+		clearInterval(this.#timer);
+		this.#taskUpdates.clear();
+	}
+};
 //#endregion
 export { PurgeLocker, Storer };