@rocicorp/zero 1.9.1-canary.0 → 1.9.1-canary.2

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts

@@ -2,16 +2,75 @@ import type { LogContext } from '@rocicorp/logger';
 import type { WatermarkedChange } from './change-streamer-service.ts';
 import type { Subscriber } from './subscriber.ts';
 export type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';
+/**
+ * Enables event-driven early release of a {@link Broadcast}. When provided and a
+ * majority has acked, the broadcast schedules its own consensus-timeout release
+ * proportional to how long the majority took to ack, instead of relying on the
+ * Forwarder's periodic progress tick.
+ */
+export type EarlyReleaseOptions = {
+    /** Time to wait, proportional to majority completion, before releasing. */
+    consensusTimeoutProportion: number;
+    /** Injectable timer, for deterministic testing. */
+    setTimeoutFn?: typeof setTimeout;
+    /** Injectable timer cancellation, for deterministic testing. */
+    clearTimeoutFn?: typeof clearTimeout;
+};
 /**
  * Initiates and tracks the progress of a change broadcasted to
- * a set of subscribers.
+ * a set of subscribers, implementing **consensus-based timeouts**
+ * to effect short-term graceful degradation when a minority of
+ * subscribers are behind.
+ *
+ * ### Background
+ *
+ * The purpose of flow control is to pull upstream replication changes
+ * no faster than the rate they are processed by downstream subscribers
+ * in the steady state. In the change-streamer, this is done by occasionally
+ * waiting for ACKs from subscribers before continuing; without doing so,
+ * I/O buffers fill up and cause the system to spend most of its time in GC.
+ *
+ * However, the naive algorithm of always waiting for all subscribers (e.g.
+ * `Promise.all()`) can behave poorly in scenarios where subscribers
+ * are imbalanced:
+ * * New subscribers may have a backlog of changes to catch up with.
+ *   Having all subscribers wait for the new subscriber to catch up results
+ *   in delaying the entire application.
+ * * Broken TCP connections similarly require all subscribers to wait until
+ *   connection liveness checks kick in and disconnect the subscriber.
  *
- * Creating a `Broadcast` automatically initiates the send.
+ * A simplistic approach is to add a limit to the amount of time waiting for
+ * subscribers, i.e. an ack timeout. However, deciding what this timeout
+ * should be is non-trivial because of the heterogeneous nature of changes;
+ * while most changes operate on single rows and are relatively predictable
+ * in terms of running time, some changes are table-wide operations and can
+ * legitimately take an arbitrary amount of time. In such scenarios, a
+ * timeout that is too short can stop progress on replication altogether.
  *
- * By default, {@link Broadcast.done} resolves when all subscribers
- * have acked the change. However, {@link Broadcast.checkProgress()}
- * can be called to resolve the broadcast earlier based on the flow
- * control policy.
+ * ### Consensus-based Timeout Algorithm
+ *
+ * To address these shortcomings, a "consensus-based timeout" algorithm is
+ * used:
+ * * Wait for more than half of the subscribers to finish. (In
+ *   case of a single node, or the case of one replication-manager
+ *   and one view-syncer, this reduces to waiting for all subscribers.)
+ * * Once more than half of the subscribers have finished, proceed after
+ *   the configured timeout, proportional to the time it took for the
+ *   majority of subscribers to ack, even if not all subscribers have
+ *   finished.
+ *
+ * In other words, the subscribers themselves are used to determine the
+ * timeout of each batch of changes; the majority determines this when
+ * they complete, upon which a timeout is logically started.
+ *
+ * In the common case, the remaining subscribers finish soon afterward and
+ * the timeout never elapses. However, in pathological cases where a minority
+ * of subscribers have a disproportionate amount of load, some will still
+ * be processing (or otherwise unresponsive). These subscribers are given
+ * a bounded amount of time to catch up at each flushed batch, up to the
+ * timeout interval. This guarantees eventual catchup because the
+ * subscribers with a backlog of changes necessarily have a higher
+ * processing rate than the subscribers that finished (and are made to wait).
  */
 export declare class Broadcast {
     #private;
@@ -24,79 +83,10 @@ export declare class Broadcast {
      * Broadcasts the `change` to the `subscribers` and tracks their
      * completion.
      */
-    constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange);
+    constructor(lc: LogContext, subscribers: Iterable<Subscriber>, change: WatermarkedChange, earlyRelease?: EarlyReleaseOptions);
     get isDone(): boolean;
     get done(): Promise<void>;
     get releaseMode(): BroadcastReleaseMode;
-    /**
-     * Checks for pathological situations in which flow should be reenabled
-     * before all subscribers have acked.
-     *
-     * ### Background
-     *
-     * The purpose of flow control is to pull upstream replication changes
-     * no faster than the rate as they are processed by downstream subscribers
-     * in the steady state. In the change-streamer, this is done by occasionally
-     * waiting for ACKs from subscribers before continuing; without doing so,
-     * I/O buffers fill up and cause the system to spend most of its time in GC.
-     *
-     * However, the naive algorithm of always waiting for all subscribers (e.g.
-     * `Promise.all()`) can behave poorly in scenarios where subscribers
-     * are imbalanced:
-     * * New subscribers may have a backlog of changes to catch up with.
-     *   Having all subscribers wait for the new subscriber to catch up results
-     *   in delaying the entire application.
-     * * Broken TCP connections similarly require all subscribers to wait until
-     *   connection liveness checks kick in and disconnect the subscriber.
-     *
-     * A simplistic approach is to add a limit to the amount of time waiting for
-     * subscribers, i.e. an ack timeout. However, deciding what this timeout
-     * should be is non-trivial because of the heterogeneous nature of changes;
-     * while most changes operate on single rows and are relatively predictable
-     * in terms of running time, some changes are table-wide operations and can
-     * legitimately take an arbitrary amount of time. In such scenarios, a
-     * timeout that is too short can stop progress on replication altogether.
-     *
-     * ### Consensus-based Timeout Algorithm
-     *
-     * To address these shortcomings, a "consensus-based timeout" algorithm is
-     * used:
-     * * Wait for more than half of the subscribers to finish. (In
-     *   case of a single node, or the case of one replication-manager
-     *   and one view-syncer, this reduces to waiting for all subscribers.)
-     * * Once more than half of the subscribers have finished, proceed after
-     *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers
-     *   have finished.
-     *
-     * In other words, the subscribers themselves are used to determine the
-     * timeout of each batch of changes; the majority determines this when
-     * they complete, upon which a timeout is logically started.
-     *
-     * In the common case, the remaining subscribers finish soon afterward and
-     * the timeout never elapses. However, in pathological cases where a minority
-     * of subscribers have a disproportionate amount of load, some will still
-     * be processing (or otherwise unresponsive). These subscribers are given
-     * a bounded amount of time to catch up at each flushed batch, up to the
-     * timeout interval. This guarantees eventual catchup because the
-     * subscribers with a backlog of changes necessarily have a higher
-     * processing rate than the subscribers that finished (and are made to wait).
-     *
-     * ### Not implemented: Broken connection detection
-     *
-     * If a subscriber has not made progress for a certain interval, the
-     * algorithm could theoretically drop it preemptively, supplementing the
-     * existing websocket-level liveness checks.
-     *
-     * However, a more reliable approach would be to change the replicator
-     * to use non-blocking writes, and subsequently increase the frequency of
-     * connection-level liveness checks. The current synchronous replica writes
-     * can delay both ping responsiveness and change progress arbitrarily (e.g.
-     * a large index creation); an independently liveness check that is not
-     * delayed by synchronous writes on the subscriber would be a more failsafe
-     * solution.
-     *
-     * @returns `true` if the broadcast was already done or was marked done.
-     */
-    checkProgress(lc: LogContext, flowControlConsensusPaddingMs: number, now: number): boolean;
+    logSlowRequests(now: number): void;
 }
 //# sourceMappingURL=broadcast.d.ts.map

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB;IAmB3B;;;OAGG;gBACS,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,iBAAiB;IAsCxE,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoEG;IACH,aAAa,CACX,EAAE,EAAE,UAAU,EACd,6BAA6B,EAAE,MAAM,EACrC,GAAG,EAAE,MAAM;CAkDd"}
+{"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E;;;;;GAKG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,2EAA2E;IAC3E,0BAA0B,EAAE,MAAM,CAAC;IACnC,mDAAmD;IACnD,YAAY,CAAC,EAAE,OAAO,UAAU,CAAC;IACjC,gEAAgE;IAChE,cAAc,CAAC,EAAE,OAAO,YAAY,CAAC;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB;IA0B3B;;;OAGG;gBAED,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB,EACzB,YAAY,CAAC,EAAE,mBAAmB;IA4EpC,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED,eAAe,CAAC,GAAG,EAAE,MAAM;CA8B5B"}

package/out/zero-cache/src/services/change-streamer/broadcast.js

@@ -2,14 +2,59 @@ import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-streamer/broadcast.ts
 /**
 * Initiates and tracks the progress of a change broadcasted to
-* a set of subscribers.
+* a set of subscribers, implementing **consensus-based timeouts**
+* to effect short-term graceful degradation when a minority of
+* subscribers are behind.
 *
-* Creating a `Broadcast` automatically initiates the send.
+* ### Background
 *
-* By default, {@link Broadcast.done} resolves when all subscribers
-* have acked the change. However, {@link Broadcast.checkProgress()}
-* can be called to resolve the broadcast earlier based on the flow
-* control policy.
+* The purpose of flow control is to pull upstream replication changes
+* no faster than the rate they are processed by downstream subscribers
+* in the steady state. In the change-streamer, this is done by occasionally
+* waiting for ACKs from subscribers before continuing; without doing so,
+* I/O buffers fill up and cause the system to spend most of its time in GC.
+*
+* However, the naive algorithm of always waiting for all subscribers (e.g.
+* `Promise.all()`) can behave poorly in scenarios where subscribers
+* are imbalanced:
+* * New subscribers may have a backlog of changes to catch up with.
+*   Having all subscribers wait for the new subscriber to catch up results
+*   in delaying the entire application.
+* * Broken TCP connections similarly require all subscribers to wait until
+*   connection liveness checks kick in and disconnect the subscriber.
+*
+* A simplistic approach is to add a limit to the amount of time waiting for
+* subscribers, i.e. an ack timeout. However, deciding what this timeout
+* should be is non-trivial because of the heterogeneous nature of changes;
+* while most changes operate on single rows and are relatively predictable
+* in terms of running time, some changes are table-wide operations and can
+* legitimately take an arbitrary amount of time. In such scenarios, a
+* timeout that is too short can stop progress on replication altogether.
+*
+* ### Consensus-based Timeout Algorithm
+*
+* To address these shortcomings, a "consensus-based timeout" algorithm is
+* used:
+* * Wait for more than half of the subscribers to finish. (In
+*   case of a single node, or the case of one replication-manager
+*   and one view-syncer, this reduces to waiting for all subscribers.)
+* * Once more than half of the subscribers have finished, proceed after
+*   the configured timeout, proportional to the time it took for the
+*   majority of subscribers to ack, even if not all subscribers have
+*   finished.
+*
+* In other words, the subscribers themselves are used to determine the
+* timeout of each batch of changes; the majority determines this when
+* they complete, upon which a timeout is logically started.
+*
+* In the common case, the remaining subscribers finish soon afterward and
+* the timeout never elapses. However, in pathological cases where a minority
+* of subscribers have a disproportionate amount of load, some will still
+* be processing (or otherwise unresponsive). These subscribers are given
+* a bounded amount of time to catch up at each flushed batch, up to the
+* timeout interval. This guarantees eventual catchup because the
+* subscribers with a backlog of changes necessarily have a higher
+* processing rate than the subscribers that finished (and are made to wait).
 */
 var Broadcast = class {
 	/**
@@ -19,6 +64,7 @@ var Broadcast = class {
 	static withoutTracking(subscribers, change) {
 		for (const sub of subscribers) sub.send(change);
 	}
+	#lc;
 	#pending;
 	#completed;
 	#done = resolver();
@@ -27,16 +73,23 @@ var Broadcast = class {
 	#watermark;
 	#majority;
 	#start = performance.now();
-	#latestCompleted = Number.MAX_VALUE;
+	#earlyReleaseTimeoutProportion;
+	#setTimeout;
+	#clearTimeout;
+	#earlyReleaseTimer;
 	/**
 	* Broadcasts the `change` to the `subscribers` and tracks their
 	* completion.
 	*/
-	constructor(subscribers, change) {
+	constructor(lc, subscribers, change, earlyRelease) {
+		this.#lc = lc;
 		this.#pending = new Set(subscribers);
 		this.#completed = [];
 		this.#watermark = change[0];
 		this.#majority = Math.floor(this.#pending.size / 2) + 1;
+		this.#earlyReleaseTimeoutProportion = earlyRelease?.consensusTimeoutProportion;
+		this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;
+		this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;
 		for (const sub of this.#pending) {
 			const changes = sub.numPending + 1;
 			sub.send(change).catch(() => {}).finally(() => this.#markCompleted(sub, changes));
@@ -44,19 +97,35 @@ var Broadcast = class {
 		if (this.#pending.size === 0) this.#setDone("all-subscribers");
 	}
 	#markCompleted(sub, changes) {
-		const elapsed = (this.#latestCompleted = performance.now()) - this.#start;
+		if (this.#isDone) return;
+		const elapsed = performance.now() - this.#start;
+		sub.trackResponseResult("on-time");
 		this.#completed.push({
 			sub,
 			changes,
 			elapsed
 		});
 		this.#pending.delete(sub);
-		if (this.#pending.size === 0) this.#setDone("all-subscribers");
+		if (this.#pending.size === 0) {
+			this.#setDone("all-subscribers");
+			return;
+		}
+		if (this.#earlyReleaseTimeoutProportion !== void 0 && this.#earlyReleaseTimer === void 0 && this.#completed.length === this.#majority) {
+			const timeoutMs = Math.ceil(elapsed * this.#earlyReleaseTimeoutProportion);
+			this.#earlyReleaseTimer = this.#setTimeout(() => this.#setDone("consensus-timeout"), timeoutMs);
+		}
 	}
 	#setDone(releaseMode) {
 		if (this.#isDone) return;
+		if (releaseMode === "consensus-timeout") {
+			const elapsed = performance.now() - this.#start;
+			this.#logWithState(`continuing with ${this.#pending.size} subscriber(s) still pending`, elapsed);
+		}
+		this.#pending.forEach((sub) => sub.trackResponseResult("timed-out"));
 		this.#isDone = true;
 		this.#releaseMode = releaseMode;
+		this.#clearTimeout(this.#earlyReleaseTimer);
+		this.#earlyReleaseTimer = void 0;
 		this.#done.resolve();
 	}
 	get isDone() {
@@ -68,92 +137,15 @@ var Broadcast = class {
 	get releaseMode() {
 		return this.#releaseMode ?? "all-subscribers";
 	}
-	/**
-	* Checks for pathological situations in which flow should be reenabled
-	* before all subscribers have acked.
-	*
-	* ### Background
-	*
-	* The purpose of flow control is to pull upstream replication changes
-	* no faster than the rate as they are processed by downstream subscribers
-	* in the steady state. In the change-streamer, this is done by occasionally
-	* waiting for ACKs from subscribers before continuing; without doing so,
-	* I/O buffers fill up and cause the system to spend most of its time in GC.
-	*
-	* However, the naive algorithm of always waiting for all subscribers (e.g.
-	* `Promise.all()`) can behave poorly in scenarios where subscribers
-	* are imbalanced:
-	* * New subscribers may have a backlog of changes to catch up with.
-	*   Having all subscribers wait for the new subscriber to catch up results
-	*   in delaying the entire application.
-	* * Broken TCP connections similarly require all subscribers to wait until
-	*   connection liveness checks kick in and disconnect the subscriber.
-	*
-	* A simplistic approach is to add a limit to the amount of time waiting for
-	* subscribers, i.e. an ack timeout. However, deciding what this timeout
-	* should be is non-trivial because of the heterogeneous nature of changes;
-	* while most changes operate on single rows and are relatively predictable
-	* in terms of running time, some changes are table-wide operations and can
-	* legitimately take an arbitrary amount of time. In such scenarios, a
-	* timeout that is too short can stop progress on replication altogether.
-	*
-	* ### Consensus-based Timeout Algorithm
-	*
-	* To address these shortcomings, a "consensus-based timeout" algorithm is
-	* used:
-	* * Wait for more than half of the subscribers to finish. (In
-	*   case of a single node, or the case of one replication-manager
-	*   and one view-syncer, this reduces to waiting for all subscribers.)
-	* * Once more than half of the subscribers have finished, proceed after
-	*   a fixed timeout elapses (e.g. 1 second), even if not all subscribers
-	*   have finished.
-	*
-	* In other words, the subscribers themselves are used to determine the
-	* timeout of each batch of changes; the majority determines this when
-	* they complete, upon which a timeout is logically started.
-	*
-	* In the common case, the remaining subscribers finish soon afterward and
-	* the timeout never elapses. However, in pathological cases where a minority
-	* of subscribers have a disproportionate amount of load, some will still
-	* be processing (or otherwise unresponsive). These subscribers are given
-	* a bounded amount of time to catch up at each flushed batch, up to the
-	* timeout interval. This guarantees eventual catchup because the
-	* subscribers with a backlog of changes necessarily have a higher
-	* processing rate than the subscribers that finished (and are made to wait).
-	*
-	* ### Not implemented: Broken connection detection
-	*
-	* If a subscriber has not made progress for a certain interval, the
-	* algorithm could theoretically drop it preemptively, supplementing the
-	* existing websocket-level liveness checks.
-	*
-	* However, a more reliable approach would be to change the replicator
-	* to use non-blocking writes, and subsequently increase the frequency of
-	* connection-level liveness checks. The current synchronous replica writes
-	* can delay both ping responsiveness and change progress arbitrarily (e.g.
-	* a large index creation); an independently liveness check that is not
-	* delayed by synchronous writes on the subscriber would be a more failsafe
-	* solution.
-	*
-	* @returns `true` if the broadcast was already done or was marked done.
-	*/
-	checkProgress(lc, flowControlConsensusPaddingMs, now) {
-		if (this.#isDone) return true;
-		if (this.#pending.size === 0) return true;
+	logSlowRequests(now) {
+		if (this.#isDone || this.#pending.size === 0) return;
 		const elapsed = now - this.#start;
 		if (this.#completed.length < this.#majority) {
-			if (elapsed >= 1e3) this.#logWithState(lc, `waiting for at least ${this.#majority} subscribers to finish`, elapsed);
-			return false;
-		}
-		if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {
-			this.#logWithState(lc, `continuing with ${this.#pending.size} subscriber(s) still pending`, elapsed);
-			this.#setDone("consensus-timeout");
-			return true;
+			if (elapsed >= 1e3) this.#logWithState(`waiting for at least ${this.#majority} subscribers to finish`, elapsed);
 		}
-		return false;
 	}
-	#logWithState(lc, msg, elapsed) {
-		lc.withContext("watermark", this.#watermark).info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {
+	#logWithState(msg, elapsed) {
+		this.#lc.withContext("watermark", this.#watermark).info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {
 			completed: this.#completed.map((d) => ({
 				id: d.sub.id,
 				processed: d.changes,

package/out/zero-cache/src/services/change-streamer/broadcast.js.map

@@ -1 +1 @@
-{"version":3,"file":"broadcast.js","names":["#pending","#completed","#done","#watermark","#majority","#start","#markCompleted","#setDone","#latestCompleted","#isDone","#releaseMode","#logWithState"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers.\n *\n * Creating a `Broadcast` automatically initiates the send.\n *\n * By default, {@link Broadcast.done} resolves when all subscribers\n * have acked the change. However, {@link Broadcast.checkProgress()}\n * can be called to resolve the broadcast earlier based on the flow\n * control policy.\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n  ) {\n    for (const sub of subscribers) {\n      void sub.send(change);\n    }\n  }\n\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n  #releaseMode: BroadcastReleaseMode | undefined;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n  #latestCompleted = Number.MAX_VALUE;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange) {\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = change[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n\n    for (const sub of this.#pending) {\n      const changes = sub.numPending + 1; // add one for this `change`\n      void sub\n        .send(change)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, changes));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    const elapsed = (this.#latestCompleted = performance.now()) - this.#start;\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n    }\n  }\n\n  #setDone(releaseMode: BroadcastReleaseMode) {\n    if (this.#isDone) {\n      return;\n    }\n    this.#isDone = true;\n    this.#releaseMode = releaseMode;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  get releaseMode(): BroadcastReleaseMode {\n    return this.#releaseMode ?? 'all-subscribers';\n  }\n\n  /**\n   * Checks for pathological situations in which flow should be reenabled\n   * before all subscribers have acked.\n   *\n   * ### Background\n   *\n   * The purpose of flow control is to pull upstream replication changes\n   * no faster than the rate as they are processed by downstream subscribers\n   * in the steady state. In the change-streamer, this is done by occasionally\n   * waiting for ACKs from subscribers before continuing; without doing so,\n   * I/O buffers fill up and cause the system to spend most of its time in GC.\n   *\n   * However, the naive algorithm of always waiting for all subscribers (e.g.\n   * `Promise.all()`) can behave poorly in scenarios where subscribers\n   * are imbalanced:\n   * * New subscribers may have a backlog of changes to catch up with.\n   *   Having all subscribers wait for the new subscriber to catch up results\n   *   in delaying the entire application.\n   * * Broken TCP connections similarly require all subscribers to wait until\n   *   connection liveness checks kick in and disconnect the subscriber.\n   *\n   * A simplistic approach is to add a limit to the amount of time waiting for\n   * subscribers, i.e. an ack timeout. However, deciding what this timeout\n   * should be is non-trivial because of the heterogeneous nature of changes;\n   * while most changes operate on single rows and are relatively predictable\n   * in terms of running time, some changes are table-wide operations and can\n   * legitimately take an arbitrary amount of time. In such scenarios, a\n   * timeout that is too short can stop progress on replication altogether.\n   *\n   * ### Consensus-based Timeout Algorithm\n   *\n   * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n   * used:\n   * * Wait for more than half of the subscribers to finish. (In\n   *   case of a single node, or the case of one replication-manager\n   *   and one view-syncer, this reduces to waiting for all subscribers.)\n   * * Once more than half of the subscribers have finished, proceed after\n   *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers\n   *   have finished.\n   *\n   * In other words, the subscribers themselves are used to determine the\n   * timeout of each batch of changes; the majority determines this when\n   * they complete, upon which a timeout is logically started.\n   *\n   * In the common case, the remaining subscribers finish soon afterward and\n   * the timeout never elapses. However, in pathological cases where a minority\n   * of subscribers have a disproportionate amount of load, some will still\n   * be processing (or otherwise unresponsive). These subscribers are given\n   * a bounded amount of time to catch up at each flushed batch, up to the\n   * timeout interval. This guarantees eventual catchup because the\n   * subscribers with a backlog of changes necessarily have a higher\n   * processing rate than the subscribers that finished (and are made to wait).\n   *\n   * ### Not implemented: Broken connection detection\n   *\n   * If a subscriber has not made progress for a certain interval, the\n   * algorithm could theoretically drop it preemptively, supplementing the\n   * existing websocket-level liveness checks.\n   *\n   * However, a more reliable approach would be to change the replicator\n   * to use non-blocking writes, and subsequently increase the frequency of\n   * connection-level liveness checks. The current synchronous replica writes\n   * can delay both ping responsiveness and change progress arbitrarily (e.g.\n   * a large index creation); an independently liveness check that is not\n   * delayed by synchronous writes on the subscriber would be a more failsafe\n   * solution.\n   *\n   * @returns `true` if the broadcast was already done or was marked done.\n   */\n  checkProgress(\n    lc: LogContext,\n    flowControlConsensusPaddingMs: number,\n    now: number,\n  ) {\n    if (this.#isDone) {\n      return true;\n    }\n    if (this.#pending.size === 0) {\n      return true;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          lc,\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n      return false;\n    }\n    // Note: In the implementation, #latestCompleted is always updated,\n    // even after the majority is reached. This is fine and does not affect\n    // the important properties of the algorithm.\n    if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {\n      this.#logWithState(\n        lc,\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n      this.#setDone('consensus-timeout');\n      return true;\n    }\n    return false;\n  }\n\n  #logWithState(lc: LogContext, msg: string, elapsed: number) {\n    lc.withContext('watermark', this.#watermark).info?.(\n      `${msg} (${elapsed.toFixed(3)} ms)`,\n      {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      },\n    );\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;AAkBA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,KAAK,MAAM,OAAO,aAChB,IAAS,KAAK,MAAM;CAExB;CAEA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CACV;CAEA;CACA;CAEA,SAAkB,YAAY,IAAI;CAClC,mBAAmB,OAAO;;;;;CAM1B,YAAY,aAAmC,QAA2B;EACxE,KAAKA,WAAW,IAAI,IAAI,WAAW;EACnC,KAAKC,aAAa,CAAC;EACnB,KAAKE,aAAa,OAAO;EACzB,KAAKC,YAAY,KAAK,MAAM,KAAKJ,SAAS,OAAO,CAAC,IAAI;EAEtD,KAAK,MAAM,OAAO,KAAKA,UAAU;GAC/B,MAAM,UAAU,IAAI,aAAa;GACjC,IACG,KAAK,MAAM,EACX,YAAY,CAAC,CAAC,EACd,cAAc,KAAKM,eAAe,KAAK,OAAO,CAAC;EACpD;EAGA,IAAI,KAAKN,SAAS,SAAS,GACzB,KAAKO,SAAS,iBAAiB;CAEnC;CAEA,eAAe,KAAiB,SAAiB;EAC/C,MAAM,WAAW,KAAKC,mBAAmB,YAAY,IAAI,KAAK,KAAKH;EACnE,KAAKJ,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAKD,SAAS,OAAO,GAAG;EACxB,IAAI,KAAKA,SAAS,SAAS,GACzB,KAAKO,SAAS,iBAAiB;CAEnC;CAEA,SAAS,aAAmC;EAC1C,IAAI,KAAKE,SACP;EAEF,KAAKA,UAAU;EACf,KAAKC,eAAe;EACpB,KAAKR,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAKO;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAKP,MAAM;CACpB;CAEA,IAAI,cAAoC;EACtC,OAAO,KAAKQ,gBAAgB;CAC9B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuEA,cACE,IACA,+BACA,KACA;EACA,IAAI,KAAKD,SACP,OAAO;EAET,IAAI,KAAKT,SAAS,SAAS,GACzB,OAAO;EAET,MAAM,UAAU,MAAM,KAAKK;EAC3B,IAAI,KAAKJ,WAAW,SAAS,KAAKG,WAAW;GAC3C,IAAI,WAAW,KACb,KAAKO,cACH,IACA,wBAAwB,KAAKP,UAAU,yBACvC,OACF;GAEF,OAAO;EACT;EAIA,IAAI,MAAM,KAAKI,oBAAoB,+BAA+B;GAChE,KAAKG,cACH,IACA,mBAAmB,KAAKX,SAAS,KAAK,+BACtC,OACF;GACA,KAAKO,SAAS,mBAAmB;GACjC,OAAO;EACT;EACA,OAAO;CACT;CAEA,cAAc,IAAgB,KAAa,SAAiB;EAC1D,GAAG,YAAY,aAAa,KAAKJ,UAAU,EAAE,OAC3C,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAC9B;GACE,WAAW,KAAKF,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAKD,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CACF;CACF;AACF"}
+{"version":3,"file":"broadcast.js","names":["#lc","#pending","#completed","#done","#watermark","#majority","#start","#earlyReleaseTimeoutProportion","#setTimeout","#clearTimeout","#markCompleted","#setDone","#isDone","#earlyReleaseTimer","#logWithState","#releaseMode"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';\n\n/**\n * Enables event-driven early release of a {@link Broadcast}. When provided and a\n * majority has acked, the broadcast schedules its own consensus-timeout release\n * proportional to how long the majority took to ack, instead of relying on the\n * Forwarder's periodic progress tick.\n */\nexport type EarlyReleaseOptions = {\n  /** Time to wait, proportional to majority completion, before releasing. */\n  consensusTimeoutProportion: number;\n  /** Injectable timer, for deterministic testing. */\n  setTimeoutFn?: typeof setTimeout;\n  /** Injectable timer cancellation, for deterministic testing. */\n  clearTimeoutFn?: typeof clearTimeout;\n};\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers, implementing **consensus-based timeouts**\n * to effect short-term graceful degradation when a minority of\n * subscribers are behind.\n *\n * ### Background\n *\n * The purpose of flow control is to pull upstream replication changes\n * no faster than the rate they are processed by downstream subscribers\n * in the steady state. In the change-streamer, this is done by occasionally\n * waiting for ACKs from subscribers before continuing; without doing so,\n * I/O buffers fill up and cause the system to spend most of its time in GC.\n *\n * However, the naive algorithm of always waiting for all subscribers (e.g.\n * `Promise.all()`) can behave poorly in scenarios where subscribers\n * are imbalanced:\n * * New subscribers may have a backlog of changes to catch up with.\n *   Having all subscribers wait for the new subscriber to catch up results\n *   in delaying the entire application.\n * * Broken TCP connections similarly require all subscribers to wait until\n *   connection liveness checks kick in and disconnect the subscriber.\n *\n * A simplistic approach is to add a limit to the amount of time waiting for\n * subscribers, i.e. an ack timeout. However, deciding what this timeout\n * should be is non-trivial because of the heterogeneous nature of changes;\n * while most changes operate on single rows and are relatively predictable\n * in terms of running time, some changes are table-wide operations and can\n * legitimately take an arbitrary amount of time. In such scenarios, a\n * timeout that is too short can stop progress on replication altogether.\n *\n * ### Consensus-based Timeout Algorithm\n *\n * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n * used:\n * * Wait for more than half of the subscribers to finish. (In\n *   case of a single node, or the case of one replication-manager\n *   and one view-syncer, this reduces to waiting for all subscribers.)\n * * Once more than half of the subscribers have finished, proceed after\n *   the configured timeout, proportional to the time it took for the\n *   majority of subscribers to ack, even if not all subscribers have\n *   finished.\n *\n * In other words, the subscribers themselves are used to determine the\n * timeout of each batch of changes; the majority determines this when\n * they complete, upon which a timeout is logically started.\n *\n * In the common case, the remaining subscribers finish soon afterward and\n * the timeout never elapses. However, in pathological cases where a minority\n * of subscribers have a disproportionate amount of load, some will still\n * be processing (or otherwise unresponsive). These subscribers are given\n * a bounded amount of time to catch up at each flushed batch, up to the\n * timeout interval. This guarantees eventual catchup because the\n * subscribers with a backlog of changes necessarily have a higher\n * processing rate than the subscribers that finished (and are made to wait).\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n  ) {\n    for (const sub of subscribers) {\n      void sub.send(change);\n    }\n  }\n\n  readonly #lc: LogContext;\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n  #releaseMode: BroadcastReleaseMode | undefined;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n\n  // When set, the broadcast schedules its own consensus-timeout release instead\n  // of waiting for the Forwarder's periodic progress tick.\n  readonly #earlyReleaseTimeoutProportion: number | undefined;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  #earlyReleaseTimer: ReturnType<typeof setTimeout> | undefined;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(\n    lc: LogContext,\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n    earlyRelease?: EarlyReleaseOptions,\n  ) {\n    this.#lc = lc;\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = change[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n    this.#earlyReleaseTimeoutProportion =\n      earlyRelease?.consensusTimeoutProportion;\n    this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;\n\n    for (const sub of this.#pending) {\n      const changes = sub.numPending + 1; // add one for this `change`\n      void sub\n        .send(change)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, changes));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    if (this.#isDone) {\n      return;\n    }\n    const elapsed = performance.now() - this.#start;\n    sub.trackResponseResult('on-time');\n\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n      return;\n    }\n    // Event-driven consensus-timeout: the moment a majority acks, arm a single\n    // release timer proportional to how long the majority took to ack. The\n    // broadcast resolves when it fires, unless all subscribers ack first.\n    if (\n      this.#earlyReleaseTimeoutProportion !== undefined &&\n      this.#earlyReleaseTimer === undefined &&\n      this.#completed.length === this.#majority\n    ) {\n      const timeoutMs = Math.ceil(\n        elapsed * this.#earlyReleaseTimeoutProportion,\n      );\n      this.#earlyReleaseTimer = this.#setTimeout(\n        () => this.#setDone('consensus-timeout'),\n        timeoutMs,\n      );\n    }\n  }\n\n  #setDone(releaseMode: BroadcastReleaseMode) {\n    if (this.#isDone) {\n      return;\n    }\n    if (releaseMode === 'consensus-timeout') {\n      const elapsed = performance.now() - this.#start;\n      this.#logWithState(\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n    }\n    this.#pending.forEach(sub => sub.trackResponseResult('timed-out'));\n    this.#isDone = true;\n    this.#releaseMode = releaseMode;\n    this.#clearTimeout(this.#earlyReleaseTimer);\n    this.#earlyReleaseTimer = undefined;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  get releaseMode(): BroadcastReleaseMode {\n    return this.#releaseMode ?? 'all-subscribers';\n  }\n\n  logSlowRequests(now: number) {\n    if (this.#isDone || this.#pending.size === 0) {\n      return;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n    }\n  }\n\n  #logWithState(msg: string, elapsed: number) {\n    this.#lc\n      .withContext('watermark', this.#watermark)\n      .info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      });\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8EA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,KAAK,MAAM,OAAO,aAChB,IAAS,KAAK,MAAM;CAExB;CAEA;CACA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CACV;CAEA;CACA;CAEA,SAAkB,YAAY,IAAI;CAIlC;CACA;CACA;CACA;;;;;CAMA,YACE,IACA,aACA,QACA,cACA;EACA,KAAKA,MAAM;EACX,KAAKC,WAAW,IAAI,IAAI,WAAW;EACnC,KAAKC,aAAa,CAAC;EACnB,KAAKE,aAAa,OAAO;EACzB,KAAKC,YAAY,KAAK,MAAM,KAAKJ,SAAS,OAAO,CAAC,IAAI;EACtD,KAAKM,iCACH,cAAc;EAChB,KAAKC,cAAc,cAAc,gBAAgB;EACjD,KAAKC,gBAAgB,cAAc,kBAAkB;EAErD,KAAK,MAAM,OAAO,KAAKR,UAAU;GAC/B,MAAM,UAAU,IAAI,aAAa;GACjC,IACG,KAAK,MAAM,EACX,YAAY,CAAC,CAAC,EACd,cAAc,KAAKS,eAAe,KAAK,OAAO,CAAC;EACpD;EAGA,IAAI,KAAKT,SAAS,SAAS,GACzB,KAAKU,SAAS,iBAAiB;CAEnC;CAEA,eAAe,KAAiB,SAAiB;EAC/C,IAAI,KAAKC,SACP;EAEF,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKN;EACzC,IAAI,oBAAoB,SAAS;EAEjC,KAAKJ,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAKD,SAAS,OAAO,GAAG;EACxB,IAAI,KAAKA,SAAS,SAAS,GAAG;GAC5B,KAAKU,SAAS,iBAAiB;GAC/B;EACF;EAIA,IACE,KAAKJ,mCAAmC,KAAA,KACxC,KAAKM,uBAAuB,KAAA,KAC5B,KAAKX,WAAW,WAAW,KAAKG,WAChC;GACA,MAAM,YAAY,KAAK,KACrB,UAAU,KAAKE,8BACjB;GACA,KAAKM,qBAAqB,KAAKL,kBACvB,KAAKG,SAAS,mBAAmB,GACvC,SACF;EACF;CACF;CAEA,SAAS,aAAmC;EAC1C,IAAI,KAAKC,SACP;EAEF,IAAI,gBAAgB,qBAAqB;GACvC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKN;GACzC,KAAKQ,cACH,mBAAmB,KAAKb,SAAS,KAAK,+BACtC,OACF;EACF;EACA,KAAKA,SAAS,SAAQ,QAAO,IAAI,oBAAoB,WAAW,CAAC;EACjE,KAAKW,UAAU;EACf,KAAKG,eAAe;EACpB,KAAKN,cAAc,KAAKI,kBAAkB;EAC1C,KAAKA,qBAAqB,KAAA;EAC1B,KAAKV,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAKS;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAKT,MAAM;CACpB;CAEA,IAAI,cAAoC;EACtC,OAAO,KAAKY,gBAAgB;CAC9B;CAEA,gBAAgB,KAAa;EAC3B,IAAI,KAAKH,WAAW,KAAKX,SAAS,SAAS,GACzC;EAEF,MAAM,UAAU,MAAM,KAAKK;EAC3B,IAAI,KAAKJ,WAAW,SAAS,KAAKG;OAC5B,WAAW,KACb,KAAKS,cACH,wBAAwB,KAAKT,UAAU,yBACvC,OACF;EAAA;CAGN;CAEA,cAAc,KAAa,SAAiB;EAC1C,KAAKL,IACF,YAAY,aAAa,KAAKI,UAAU,EACxC,OAAO,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAAO;GAC3C,WAAW,KAAKF,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAKD,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CAAC;CACL;AACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts

@@ -8,7 +8,8 @@ import type { SubscriptionState } from '../replicator/schema/replication-state.t
 import { type ChangeStreamerService } from './change-streamer.ts';
 import { type PurgeLock, type TuningOptions as StorerOptions } from './storer.ts';
 export type TuningOptions = StorerOptions & {
-    flowControlConsensusPaddingSeconds: number;
+    flowControlConsensusTimeoutProportion: number;
+    flowControlSlowSubscriberGracePeriodMs?: number | undefined;
 };
 /**
  * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EAEtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAGrB,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC;AAID,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAYjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EAEtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAGrB,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7D,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC;AAID,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -1,5 +1,6 @@
 import { unreachable } from "../../../../shared/src/asserts.js";
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { promiseOrAbort } from "../../../../shared/src/promise-race.js";
 import "../change-source/protocol/current/downstream.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
 import { DEFAULT_MAX_RETRY_DELAY_MS, RunningState, UnrecoverableError } from "../running-state.js";
@@ -195,7 +196,10 @@ var ChangeStreamerImpl = class {
 			consumed[1],
 			consumed[2]
 		]), (err) => this.stop(err), opts);
-		this.#forwarder = new Forwarder(lc, { flowControlConsensusPaddingSeconds: opts.flowControlConsensusPaddingSeconds });
+		this.#forwarder = new Forwarder(lc, {
+			flowControlConsensusTimeoutProportion: opts.flowControlConsensusTimeoutProportion,
+			flowControlSlowSubscriberGracePeriodMs: opts.flowControlSlowSubscriberGracePeriodMs
+		});
 		this.#replicationStatusPublisher = replicationStatusPublisher;
 		this.#purgeLock = initialPurgeLock;
 		this.#autoReset = autoReset;
@@ -255,13 +259,12 @@ var ChangeStreamerImpl = class {
 					unflushedBytes += json.length;
 					if (unflushedBytes < flushBytesThreshold) this.#forwarder.forward(entry);
 					else {
-						const forwarded = this.#forwarder.forwardWithFlowControl(entry);
-						await stream.changes.doneOr(forwarded);
+						await promiseOrAbort(this.#forwarder.forwardWithFlowControl(entry), stream.changes.signal, this.#state.signal);
 						unflushedBytes = 0;
 					}
 					if (type === "commit" || type === "rollback") watermark = null;
 					const readyForMore = this.#storer.readyForMore();
-					if (readyForMore) await stream.changes.doneOr(readyForMore);
+					if (readyForMore) await promiseOrAbort(readyForMore, stream.changes.signal, this.#state.signal);
 				}
 			} catch (e) {
 				err = e;