@rocicorp/zero 0.26.1 → 0.26.2-canary.4

package/out/zero-cache/src/services/change-streamer/broadcast.js

@@ -1,171 +1,165 @@
 import { resolver } from "@rocicorp/resolver";
-class Broadcast {
-  /**
-   * Sends the change to the subscribers without the tracking machinery.
-   * This is suitable for fire-and-forget (i.e. pipelined) sends.
-   */
-  static withoutTracking(subscribers, change) {
-    for (const sub of subscribers) {
-      void sub.send(change);
-    }
-  }
-  #pending;
-  #completed;
-  #done = resolver();
-  #isDone = false;
-  #watermark;
-  #majority;
-  #start = performance.now();
-  #latestCompleted = Number.MAX_VALUE;
-  /**
-   * Broadcasts the `change` to the `subscribers` and tracks their
-   * completion.
-   */
-  constructor(subscribers, change) {
-    this.#pending = new Set(subscribers);
-    this.#completed = [];
-    this.#watermark = change[0];
-    this.#majority = Math.floor(this.#pending.size / 2) + 1;
-    for (const sub of this.#pending) {
-      const changes = sub.numPending + 1;
-      void sub.send(change).catch(() => {
-      }).finally(() => this.#markCompleted(sub, changes));
-    }
-    if (this.#pending.size === 0) {
-      this.#setDone();
-    }
-  }
-  #markCompleted(sub, changes) {
-    const elapsed = (this.#latestCompleted = performance.now()) - this.#start;
-    this.#completed.push({ sub, changes, elapsed });
-    this.#pending.delete(sub);
-    if (this.#pending.size === 0) {
-      this.#setDone();
-    }
-  }
-  #setDone() {
-    this.#isDone = true;
-    this.#done.resolve();
-  }
-  get isDone() {
-    return this.#isDone;
-  }
-  get done() {
-    return this.#done.promise;
-  }
-  /**
-   * Checks for pathological situations in which flow should be reenabled
-   * before all subscribers have acked.
-   *
-   * ### Background
-   *
-   * The purpose of flow control is to pull upstream replication changes
-   * no faster than the rate as they are processed by downstream subscribers
-   * in the steady state. In the change-streamer, this is done by occasionally
-   * waiting for ACKs from subscribers before continuing; without doing so,
-   * I/O buffers fill up and cause the system to spend most of its time in GC.
-   *
-   * However, the naive algorithm of always waiting for all subscribers (e.g.
-   * `Promise.all()`) can behave poorly in scenarios where subscribers
-   * are imbalanced:
-   * * New subscribers may have a backlog of changes to catch up with.
-   *   Having all subscribers wait for the new subscriber to catch up results
-   *   in delaying the entire application.
-   * * Broken TCP connections similarly require all subscribers to wait until
-   *   connection liveness checks kick in and disconnect the subscriber.
-   *
-   * A simplistic approach is to add a limit to the amount of time waiting for
-   * subscribers, i.e. an ack timeout. However, deciding what this timeout
-   * should be is non-trivial because of the heterogeneous nature of changes;
-   * while most changes operate on single rows and are relatively predictable
-   * in terms of running time, some changes are table-wide operations and can
-   * legitimately take an arbitrary amount of time. In such scenarios, a
-   * timeout that is too short can stop progress on replication altogether.
-   *
-   * ### Consensus-based Timeout Algorithm
-   *
-   * To address these shortcomings, a "consensus-based timeout" algorithm is
-   * used:
-   * * Wait for more than half of the subscribers to finish. (In
-   *   case of a single node, or the case of one replication-manager
-   *   and one view-syncer, this reduces to waiting for all subscribers.)
-   * * Once more than half of the subscribers have finished, proceed after
-   *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers
-   *   have finished.
-   *
-   * In other words, the subscribers themselves are used to determine the
-   * timeout of each batch of changes; the majority determines this when
-   * they complete, upon which a timeout is logically started.
-   *
-   * In the common case, the remaining subscribers finish soon afterward and
-   * the timeout never elapses. However, in pathological cases where a minority
-   * of subscribers have a disproportionate amount of load, some will still
-   * be processing (or otherwise unresponsive). These subscribers are given
-   * a bounded amount of time to catch up at each flushed batch, up to the
-   * timeout interval. This guarantees eventual catchup because the
-   * subscribers with a backlog of changes necessarily have a higher
-   * processing rate than the subscribers that finished (and are made to wait).
-   *
-   * ### Not implemented: Broken connection detection
-   *
-   * If a subscriber has not made progress for a certain interval, the
-   * algorithm could theoretically drop it preemptively, supplementing the
-   * existing websocket-level liveness checks.
-   *
-   * However, a more reliable approach would be to change the replicator
-   * to use non-blocking writes, and subsequently increase the frequency of
-   * connection-level liveness checks. The current synchronous replica writes
-   * can delay both ping responsiveness and change progress arbitrarily (e.g.
-   * a large index creation); an independently liveness check that is not
-   * delayed by synchronous writes on the subscriber would be a more failsafe
-   * solution.
-   *
-   * @returns `true` if the broadcast was already done or was marked done.
-   */
-  checkProgress(lc, flowControlConsensusPaddingMs, now) {
-    if (this.#pending.size === 0) {
-      return true;
-    }
-    const elapsed = now - this.#start;
-    if (this.#completed.length < this.#majority) {
-      if (elapsed >= 1e3) {
-        this.#logWithState(
-          lc,
-          `waiting for at least ${this.#majority} subscribers to finish`,
-          elapsed
-        );
-      }
-      return false;
-    }
-    if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {
-      this.#logWithState(
-        lc,
-        `continuing with ${this.#pending.size} subscriber(s) still pending`,
-        elapsed
-      );
-      this.#setDone();
-      return true;
-    }
-    return false;
-  }
-  #logWithState(lc, msg, elapsed) {
-    lc.withContext("watermark", this.#watermark).info?.(
-      `${msg} (${elapsed.toFixed(3)} ms)`,
-      {
-        completed: this.#completed.map((d) => ({
-          id: d.sub.id,
-          processed: d.changes,
-          elapsed: d.elapsed
-        })),
-        pending: [...this.#pending].map((sub) => ({
-          id: sub.id,
-          ...sub.getStats()
-        }))
-      }
-    );
-  }
-}
-export {
-  Broadcast
+//#region ../zero-cache/src/services/change-streamer/broadcast.ts
+/**
+* Initiates and tracks the progress of a change broadcasted to
+* a set of subscribers.
+*
+* Creating a `Broadcast` automatically initiates the send.
+*
+* By default, {@link Broadcast.done} resolves when all subscribers
+* have acked the change. However, {@link Broadcast.checkProgress()}
+* can be called to resolve the broadcast earlier based on the flow
+* control policy.
+*/
+var Broadcast = class {
+	/**
+	* Sends the change to the subscribers without the tracking machinery.
+	* This is suitable for fire-and-forget (i.e. pipelined) sends.
+	*/
+	static withoutTracking(subscribers, change) {
+		for (const sub of subscribers) sub.send(change);
+	}
+	#pending;
+	#completed;
+	#done = resolver();
+	#isDone = false;
+	#watermark;
+	#majority;
+	#start = performance.now();
+	#latestCompleted = Number.MAX_VALUE;
+	/**
+	* Broadcasts the `change` to the `subscribers` and tracks their
+	* completion.
+	*/
+	constructor(subscribers, change) {
+		this.#pending = new Set(subscribers);
+		this.#completed = [];
+		this.#watermark = change[0];
+		this.#majority = Math.floor(this.#pending.size / 2) + 1;
+		for (const sub of this.#pending) {
+			const changes = sub.numPending + 1;
+			sub.send(change).catch(() => {}).finally(() => this.#markCompleted(sub, changes));
+		}
+		if (this.#pending.size === 0) this.#setDone();
+	}
+	#markCompleted(sub, changes) {
+		const elapsed = (this.#latestCompleted = performance.now()) - this.#start;
+		this.#completed.push({
+			sub,
+			changes,
+			elapsed
+		});
+		this.#pending.delete(sub);
+		if (this.#pending.size === 0) this.#setDone();
+	}
+	#setDone() {
+		this.#isDone = true;
+		this.#done.resolve();
+	}
+	get isDone() {
+		return this.#isDone;
+	}
+	get done() {
+		return this.#done.promise;
+	}
+	/**
+	* Checks for pathological situations in which flow should be reenabled
+	* before all subscribers have acked.
+	*
+	* ### Background
+	*
+	* The purpose of flow control is to pull upstream replication changes
+	* no faster than the rate as they are processed by downstream subscribers
+	* in the steady state. In the change-streamer, this is done by occasionally
+	* waiting for ACKs from subscribers before continuing; without doing so,
+	* I/O buffers fill up and cause the system to spend most of its time in GC.
+	*
+	* However, the naive algorithm of always waiting for all subscribers (e.g.
+	* `Promise.all()`) can behave poorly in scenarios where subscribers
+	* are imbalanced:
+	* * New subscribers may have a backlog of changes to catch up with.
+	*   Having all subscribers wait for the new subscriber to catch up results
+	*   in delaying the entire application.
+	* * Broken TCP connections similarly require all subscribers to wait until
+	*   connection liveness checks kick in and disconnect the subscriber.
+	*
+	* A simplistic approach is to add a limit to the amount of time waiting for
+	* subscribers, i.e. an ack timeout. However, deciding what this timeout
+	* should be is non-trivial because of the heterogeneous nature of changes;
+	* while most changes operate on single rows and are relatively predictable
+	* in terms of running time, some changes are table-wide operations and can
+	* legitimately take an arbitrary amount of time. In such scenarios, a
+	* timeout that is too short can stop progress on replication altogether.
+	*
+	* ### Consensus-based Timeout Algorithm
+	*
+	* To address these shortcomings, a "consensus-based timeout" algorithm is
+	* used:
+	* * Wait for more than half of the subscribers to finish. (In
+	*   case of a single node, or the case of one replication-manager
+	*   and one view-syncer, this reduces to waiting for all subscribers.)
+	* * Once more than half of the subscribers have finished, proceed after
+	*   a fixed timeout elapses (e.g. 1 second), even if not all subscribers
+	*   have finished.
+	*
+	* In other words, the subscribers themselves are used to determine the
+	* timeout of each batch of changes; the majority determines this when
+	* they complete, upon which a timeout is logically started.
+	*
+	* In the common case, the remaining subscribers finish soon afterward and
+	* the timeout never elapses. However, in pathological cases where a minority
+	* of subscribers have a disproportionate amount of load, some will still
+	* be processing (or otherwise unresponsive). These subscribers are given
+	* a bounded amount of time to catch up at each flushed batch, up to the
+	* timeout interval. This guarantees eventual catchup because the
+	* subscribers with a backlog of changes necessarily have a higher
+	* processing rate than the subscribers that finished (and are made to wait).
+	*
+	* ### Not implemented: Broken connection detection
+	*
+	* If a subscriber has not made progress for a certain interval, the
+	* algorithm could theoretically drop it preemptively, supplementing the
+	* existing websocket-level liveness checks.
+	*
+	* However, a more reliable approach would be to change the replicator
+	* to use non-blocking writes, and subsequently increase the frequency of
+	* connection-level liveness checks. The current synchronous replica writes
+	* can delay both ping responsiveness and change progress arbitrarily (e.g.
+	* a large index creation); an independently liveness check that is not
+	* delayed by synchronous writes on the subscriber would be a more failsafe
+	* solution.
+	*
+	* @returns `true` if the broadcast was already done or was marked done.
+	*/
+	checkProgress(lc, flowControlConsensusPaddingMs, now) {
+		if (this.#pending.size === 0) return true;
+		const elapsed = now - this.#start;
+		if (this.#completed.length < this.#majority) {
+			if (elapsed >= 1e3) this.#logWithState(lc, `waiting for at least ${this.#majority} subscribers to finish`, elapsed);
+			return false;
+		}
+		if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {
+			this.#logWithState(lc, `continuing with ${this.#pending.size} subscriber(s) still pending`, elapsed);
+			this.#setDone();
+			return true;
+		}
+		return false;
+	}
+	#logWithState(lc, msg, elapsed) {
+		lc.withContext("watermark", this.#watermark).info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {
+			completed: this.#completed.map((d) => ({
+				id: d.sub.id,
+				processed: d.changes,
+				elapsed: d.elapsed
+			})),
+			pending: [...this.#pending].map((sub) => ({
+				id: sub.id,
+				...sub.getStats()
+			}))
+		});
+	}
 };
-//# sourceMappingURL=broadcast.js.map
+//#endregion
+export { Broadcast };
+
+//# sourceMappingURL=broadcast.js.map

package/out/zero-cache/src/services/change-streamer/broadcast.js.map

@@ -1 +1 @@
-{"version":3,"file":"broadcast.js","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers.\n *\n * Creating a `Broadcast` automatically initiates the send.\n *\n * By default, {@link Broadcast.done} resolves when all subscribers\n * have acked the change. However, {@link Broadcast.checkProgress()}\n * can be called to resolve the broadcast earlier based on the flow\n * control policy.\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n  ) {\n    for (const sub of subscribers) {\n      void sub.send(change);\n    }\n  }\n\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n  #latestCompleted = Number.MAX_VALUE;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange) {\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = change[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n\n    for (const sub of this.#pending) {\n      const changes = sub.numPending + 1; // add one for this `change`\n      void sub\n        .send(change)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, changes));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone();\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    const elapsed = (this.#latestCompleted = performance.now()) - this.#start;\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone();\n    }\n  }\n\n  #setDone() {\n    this.#isDone = true;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  /**\n   * Checks for pathological situations in which flow should be reenabled\n   * before all subscribers have acked.\n   *\n   * ### Background\n   *\n   * The purpose of flow control is to pull upstream replication changes\n   * no faster than the rate as they are processed by downstream subscribers\n   * in the steady state. In the change-streamer, this is done by occasionally\n   * waiting for ACKs from subscribers before continuing; without doing so,\n   * I/O buffers fill up and cause the system to spend most of its time in GC.\n   *\n   * However, the naive algorithm of always waiting for all subscribers (e.g.\n   * `Promise.all()`) can behave poorly in scenarios where subscribers\n   * are imbalanced:\n   * * New subscribers may have a backlog of changes to catch up with.\n   *   Having all subscribers wait for the new subscriber to catch up results\n   *   in delaying the entire application.\n   * * Broken TCP connections similarly require all subscribers to wait until\n   *   connection liveness checks kick in and disconnect the subscriber.\n   *\n   * A simplistic approach is to add a limit to the amount of time waiting for\n   * subscribers, i.e. an ack timeout. However, deciding what this timeout\n   * should be is non-trivial because of the heterogeneous nature of changes;\n   * while most changes operate on single rows and are relatively predictable\n   * in terms of running time, some changes are table-wide operations and can\n   * legitimately take an arbitrary amount of time. In such scenarios, a\n   * timeout that is too short can stop progress on replication altogether.\n   *\n   * ### Consensus-based Timeout Algorithm\n   *\n   * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n   * used:\n   * * Wait for more than half of the subscribers to finish. (In\n   *   case of a single node, or the case of one replication-manager\n   *   and one view-syncer, this reduces to waiting for all subscribers.)\n   * * Once more than half of the subscribers have finished, proceed after\n   *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers\n   *   have finished.\n   *\n   * In other words, the subscribers themselves are used to determine the\n   * timeout of each batch of changes; the majority determines this when\n   * they complete, upon which a timeout is logically started.\n   *\n   * In the common case, the remaining subscribers finish soon afterward and\n   * the timeout never elapses. However, in pathological cases where a minority\n   * of subscribers have a disproportionate amount of load, some will still\n   * be processing (or otherwise unresponsive). These subscribers are given\n   * a bounded amount of time to catch up at each flushed batch, up to the\n   * timeout interval. This guarantees eventual catchup because the\n   * subscribers with a backlog of changes necessarily have a higher\n   * processing rate than the subscribers that finished (and are made to wait).\n   *\n   * ### Not implemented: Broken connection detection\n   *\n   * If a subscriber has not made progress for a certain interval, the\n   * algorithm could theoretically drop it preemptively, supplementing the\n   * existing websocket-level liveness checks.\n   *\n   * However, a more reliable approach would be to change the replicator\n   * to use non-blocking writes, and subsequently increase the frequency of\n   * connection-level liveness checks. The current synchronous replica writes\n   * can delay both ping responsiveness and change progress arbitrarily (e.g.\n   * a large index creation); an independently liveness check that is not\n   * delayed by synchronous writes on the subscriber would be a more failsafe\n   * solution.\n   *\n   * @returns `true` if the broadcast was already done or was marked done.\n   */\n  checkProgress(\n    lc: LogContext,\n    flowControlConsensusPaddingMs: number,\n    now: number,\n  ) {\n    if (this.#pending.size === 0) {\n      return true;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          lc,\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n      return false;\n    }\n    // Note: In the implementation, #latestCompleted is always updated,\n    // even after the majority is reached. This is fine and does not affect\n    // the important properties of the algorithm.\n    if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {\n      this.#logWithState(\n        lc,\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n      this.#setDone();\n      return true;\n    }\n    return false;\n  }\n\n  #logWithState(lc: LogContext, msg: string, elapsed: number) {\n    lc.withContext('watermark', this.#watermark).info?.(\n      `${msg} (${elapsed.toFixed(3)} ms)`,\n      {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: [...this.#pending].map(sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      },\n    );\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n"],"names":[],"mappings":";AAgBO,MAAM,UAAU;AAAA;AAAA;AAAA;AAAA;AAAA,EAKrB,OAAO,gBACL,aACA,QACA;AACA,eAAW,OAAO,aAAa;AAC7B,WAAK,IAAI,KAAK,MAAM;AAAA,IACtB;AAAA,EACF;AAAA,EAES;AAAA,EACA;AAAA,EACA,QAAQ,SAAA;AAAA,EACjB,UAAU;AAAA,EAED;AAAA,EACA;AAAA,EAEA,SAAS,YAAY,IAAA;AAAA,EAC9B,mBAAmB,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA,EAM1B,YAAY,aAAmC,QAA2B;AACxE,SAAK,WAAW,IAAI,IAAI,WAAW;AACnC,SAAK,aAAa,CAAA;AAClB,SAAK,aAAa,OAAO,CAAC;AAC1B,SAAK,YAAY,KAAK,MAAM,KAAK,SAAS,OAAO,CAAC,IAAI;AAEtD,eAAW,OAAO,KAAK,UAAU;AAC/B,YAAM,UAAU,IAAI,aAAa;AACjC,WAAK,IACF,KAAK,MAAM,EACX,MAAM,MAAM;AAAA,MAAC,CAAC,EACd,QAAQ,MAAM,KAAK,eAAe,KAAK,OAAO,CAAC;AAAA,IACpD;AAGA,QAAI,KAAK,SAAS,SAAS,GAAG;AAC5B,WAAK,SAAA;AAAA,IACP;AAAA,EACF;AAAA,EAEA,eAAe,KAAiB,SAAiB;AAC/C,UAAM,WAAW,KAAK,mBAAmB,YAAY,IAAA,KAAS,KAAK;AACnE,SAAK,WAAW,KAAK,EAAC,KAAK,SAAS,SAAQ;AAC5C,SAAK,SAAS,OAAO,GAAG;AACxB,QAAI,KAAK,SAAS,SAAS,GAAG;AAC5B,WAAK,SAAA;AAAA,IACP;AAAA,EACF;AAAA,EAEA,WAAW;AACT,SAAK,UAAU;AACf,SAAK,MAAM,QAAA;AAAA,EACb;AAAA,EAEA,IAAI,SAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAsB;AACxB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAuEA,cACE,IACA,+BACA,KACA;AACA,QAAI,KAAK,SAAS,SAAS,GAAG;AAC5B,aAAO;AAAA,IACT;AACA,UAAM,UAAU,MAAM,KAAK;AAC3B,QAAI,KAAK,WAAW,SAAS,KAAK,WAAW;AAC3C,UAAI,WAAW,KAAM;AACnB,aAAK;AAAA,UACH;AAAA,UACA,wBAAwB,KAAK,SAAS;AAAA,UACtC;AAAA,QAAA;AAAA,MAEJ;AACA,aAAO;AAAA,IACT;AAIA,QAAI,MAAM,KAAK,oBAAoB,+BAA+B;AAChE,WAAK;AAAA,QACH;AAAA,QACA,mBAAmB,KAAK,SAAS,IAAI;AAAA,QACrC;AAAA,MAAA;AAEF,WAAK,SAAA;AACL,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAAA,EAEA,cAAc,IAAgB,KAAa,SAAiB;AAC1D,OAAG,YAAY,aAAa,KAAK,UAAU,EAAE;AAAA,MAC3C,GAAG,GAAG,KAAK,QAAQ,QAAQ,CAAC,CAAC;AAAA,MAC7B;AAAA,QACE,WAAW,KAAK,WAAW,IAAI,CAAA,OAAM;AAAA,UACnC,IAAI,EAAE,IAAI;AAAA,UACV,WAAW,EAAE;AAAA,UACb,SAAS,EAAE;AAAA,QAAA,EACX;AAAA,QACF,SAAS,CAAC,GAAG,KAAK,QAAQ,EAAE,IAAI,CAAA,SAAQ;AAAA,UACtC,IAAI,IAAI;AAAA,UACR,GAAG,IAAI,SAAA;AAAA,QAAS,EAChB;AAAA,MAAA;AAAA,IACJ;AAAA,EAEJ;AACF;"}
+{"version":3,"file":"broadcast.js","names":["#pending","#completed","#done","#watermark","#majority","#start","#markCompleted","#setDone","#latestCompleted","#isDone","#logWithState"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers.\n *\n * Creating a `Broadcast` automatically initiates the send.\n *\n * By default, {@link Broadcast.done} resolves when all subscribers\n * have acked the change. However, {@link Broadcast.checkProgress()}\n * can be called to resolve the broadcast earlier based on the flow\n * control policy.\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n  ) {\n    for (const sub of subscribers) {\n      void sub.send(change);\n    }\n  }\n\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n  #latestCompleted = Number.MAX_VALUE;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange) {\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = change[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n\n    for (const sub of this.#pending) {\n      const changes = sub.numPending + 1; // add one for this `change`\n      void sub\n        .send(change)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, changes));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone();\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    const elapsed = (this.#latestCompleted = performance.now()) - this.#start;\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone();\n    }\n  }\n\n  #setDone() {\n    this.#isDone = true;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  /**\n   * Checks for pathological situations in which flow should be reenabled\n   * before all subscribers have acked.\n   *\n   * ### Background\n   *\n   * The purpose of flow control is to pull upstream replication changes\n   * no faster than the rate as they are processed by downstream subscribers\n   * in the steady state. In the change-streamer, this is done by occasionally\n   * waiting for ACKs from subscribers before continuing; without doing so,\n   * I/O buffers fill up and cause the system to spend most of its time in GC.\n   *\n   * However, the naive algorithm of always waiting for all subscribers (e.g.\n   * `Promise.all()`) can behave poorly in scenarios where subscribers\n   * are imbalanced:\n   * * New subscribers may have a backlog of changes to catch up with.\n   *   Having all subscribers wait for the new subscriber to catch up results\n   *   in delaying the entire application.\n   * * Broken TCP connections similarly require all subscribers to wait until\n   *   connection liveness checks kick in and disconnect the subscriber.\n   *\n   * A simplistic approach is to add a limit to the amount of time waiting for\n   * subscribers, i.e. an ack timeout. However, deciding what this timeout\n   * should be is non-trivial because of the heterogeneous nature of changes;\n   * while most changes operate on single rows and are relatively predictable\n   * in terms of running time, some changes are table-wide operations and can\n   * legitimately take an arbitrary amount of time. In such scenarios, a\n   * timeout that is too short can stop progress on replication altogether.\n   *\n   * ### Consensus-based Timeout Algorithm\n   *\n   * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n   * used:\n   * * Wait for more than half of the subscribers to finish. (In\n   *   case of a single node, or the case of one replication-manager\n   *   and one view-syncer, this reduces to waiting for all subscribers.)\n   * * Once more than half of the subscribers have finished, proceed after\n   *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers\n   *   have finished.\n   *\n   * In other words, the subscribers themselves are used to determine the\n   * timeout of each batch of changes; the majority determines this when\n   * they complete, upon which a timeout is logically started.\n   *\n   * In the common case, the remaining subscribers finish soon afterward and\n   * the timeout never elapses. However, in pathological cases where a minority\n   * of subscribers have a disproportionate amount of load, some will still\n   * be processing (or otherwise unresponsive). These subscribers are given\n   * a bounded amount of time to catch up at each flushed batch, up to the\n   * timeout interval. This guarantees eventual catchup because the\n   * subscribers with a backlog of changes necessarily have a higher\n   * processing rate than the subscribers that finished (and are made to wait).\n   *\n   * ### Not implemented: Broken connection detection\n   *\n   * If a subscriber has not made progress for a certain interval, the\n   * algorithm could theoretically drop it preemptively, supplementing the\n   * existing websocket-level liveness checks.\n   *\n   * However, a more reliable approach would be to change the replicator\n   * to use non-blocking writes, and subsequently increase the frequency of\n   * connection-level liveness checks. The current synchronous replica writes\n   * can delay both ping responsiveness and change progress arbitrarily (e.g.\n   * a large index creation); an independently liveness check that is not\n   * delayed by synchronous writes on the subscriber would be a more failsafe\n   * solution.\n   *\n   * @returns `true` if the broadcast was already done or was marked done.\n   */\n  checkProgress(\n    lc: LogContext,\n    flowControlConsensusPaddingMs: number,\n    now: number,\n  ) {\n    if (this.#pending.size === 0) {\n      return true;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          lc,\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n      return false;\n    }\n    // Note: In the implementation, #latestCompleted is always updated,\n    // even after the majority is reached. This is fine and does not affect\n    // the important properties of the algorithm.\n    if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {\n      this.#logWithState(\n        lc,\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n      this.#setDone();\n      return true;\n    }\n    return false;\n  }\n\n  #logWithState(lc: LogContext, msg: string, elapsed: number) {\n    lc.withContext('watermark', this.#watermark).info?.(\n      `${msg} (${elapsed.toFixed(3)} ms)`,\n      {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: [...this.#pending].map(sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      },\n    );\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;AAgBA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;AACA,OAAK,MAAM,OAAO,YACX,KAAI,KAAK,OAAO;;CAIzB;CACA;CACA,QAAiB,UAAU;CAC3B,UAAU;CAEV;CACA;CAEA,SAAkB,YAAY,KAAK;CACnC,mBAAmB,OAAO;;;;;CAM1B,YAAY,aAAmC,QAA2B;AACxE,QAAA,UAAgB,IAAI,IAAI,YAAY;AACpC,QAAA,YAAkB,EAAE;AACpB,QAAA,YAAkB,OAAO;AACzB,QAAA,WAAiB,KAAK,MAAM,MAAA,QAAc,OAAO,EAAE,GAAG;AAEtD,OAAK,MAAM,OAAO,MAAA,SAAe;GAC/B,MAAM,UAAU,IAAI,aAAa;AAC5B,OACF,KAAK,OAAO,CACZ,YAAY,GAAG,CACf,cAAc,MAAA,cAAoB,KAAK,QAAQ,CAAC;;AAIrD,MAAI,MAAA,QAAc,SAAS,EACzB,OAAA,SAAe;;CAInB,eAAe,KAAiB,SAAiB;EAC/C,MAAM,WAAW,MAAA,kBAAwB,YAAY,KAAK,IAAI,MAAA;AAC9D,QAAA,UAAgB,KAAK;GAAC;GAAK;GAAS;GAAQ,CAAC;AAC7C,QAAA,QAAc,OAAO,IAAI;AACzB,MAAI,MAAA,QAAc,SAAS,EACzB,OAAA,SAAe;;CAInB,WAAW;AACT,QAAA,SAAe;AACf,QAAA,KAAW,SAAS;;CAGtB,IAAI,SAAkB;AACpB,SAAO,MAAA;;CAGT,IAAI,OAAsB;AACxB,SAAO,MAAA,KAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwEpB,cACE,IACA,+BACA,KACA;AACA,MAAI,MAAA,QAAc,SAAS,EACzB,QAAO;EAET,MAAM,UAAU,MAAM,MAAA;AACtB,MAAI,MAAA,UAAgB,SAAS,MAAA,UAAgB;AAC3C,OAAI,WAAW,IACb,OAAA,aACE,IACA,wBAAwB,MAAA,SAAe,yBACvC,QACD;AAEH,UAAO;;AAKT,MAAI,MAAM,MAAA,mBAAyB,+BAA+B;AAChE,SAAA,aACE,IACA,mBAAmB,MAAA,QAAc,KAAK,+BACtC,QACD;AACD,SAAA,SAAe;AACf,UAAO;;AAET,SAAO;;CAGT,cAAc,IAAgB,KAAa,SAAiB;AAC1D,KAAG,YAAY,aAAa,MAAA,UAAgB,CAAC,OAC3C,GAAG,IAAI,IAAI,QAAQ,QAAQ,EAAE,CAAC,OAC9B;GACE,WAAW,MAAA,UAAgB,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;IACZ,EAAE;GACH,SAAS,CAAC,GAAG,MAAA,QAAc,CAAC,KAAI,SAAQ;IACtC,IAAI,IAAI;IACR,GAAG,IAAI,UAAU;IAClB,EAAE;GACJ,CACF"}