@rocicorp/zero 0.26.0 → 0.26.1-canary.11

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts

@@ -71,6 +71,11 @@ declare const data: v.TupleType<[v.Type<"data">, v.UnionType<[v.UnionType<[v.Obj
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
     rowValues: v.ArrayType<v.ArrayType<v.Type<import("../../../../../../shared/src/bigint-json.ts").JSONValue>>>;
+    status: v.Optional<{
+        totalBytes?: number | undefined;
+        rows: number;
+        totalRows: number;
+    }>;
 }, undefined>]>, v.UnionType<[v.ObjectType<{
     tag: v.Type<"create-table">;
     spec: v.ObjectType<Omit<{
@@ -209,6 +214,11 @@ declare const data: v.TupleType<[v.Type<"data">, v.UnionType<[v.UnionType<[v.Obj
     }, undefined>;
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
+    status: v.Optional<{
+        totalBytes?: number | undefined;
+        rows: number;
+        totalRows: number;
+    }>;
 }, undefined>]>]>]>;
 declare const commit: v.TupleType<[v.Type<"commit">, v.ObjectType<{
     tag: v.Type<"commit">;
@@ -293,6 +303,11 @@ export declare const changeStreamDataSchema: v.UnionType<[v.TupleType<[v.Type<"b
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
     rowValues: v.ArrayType<v.ArrayType<v.Type<import("../../../../../../shared/src/bigint-json.ts").JSONValue>>>;
+    status: v.Optional<{
+        totalBytes?: number | undefined;
+        rows: number;
+        totalRows: number;
+    }>;
 }, undefined>]>, v.UnionType<[v.ObjectType<{
     tag: v.Type<"create-table">;
     spec: v.ObjectType<Omit<{
@@ -431,6 +446,11 @@ export declare const changeStreamDataSchema: v.UnionType<[v.TupleType<[v.Type<"b
     }, undefined>;
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
+    status: v.Optional<{
+        totalBytes?: number | undefined;
+        rows: number;
+        totalRows: number;
+    }>;
 }, undefined>]>]>]>, v.TupleType<[v.Type<"commit">, v.ObjectType<{
     tag: v.Type<"commit">;
 }, undefined>, v.ObjectType<{
@@ -517,6 +537,11 @@ export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.Tupl
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
     rowValues: v.ArrayType<v.ArrayType<v.Type<import("../../../../../../shared/src/bigint-json.ts").JSONValue>>>;
+    status: v.Optional<{
+        totalBytes?: number | undefined;
+        rows: number;
+        totalRows: number;
+    }>;
 }, undefined>]>, v.UnionType<[v.ObjectType<{
     tag: v.Type<"create-table">;
     spec: v.ObjectType<Omit<{
@@ -655,6 +680,11 @@ export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.Tupl
     }, undefined>;
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
+    status: v.Optional<{
+        totalBytes?: number | undefined;
+        rows: number;
+        totalRows: number;
+    }>;
 }, undefined>]>]>]>, v.TupleType<[v.Type<"commit">, v.ObjectType<{
     tag: v.Type<"commit">;
 }, undefined>, v.ObjectType<{

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
+{"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current.js

@@ -1,5 +1,5 @@
 import { resetRequiredSchema } from "./current/control.js";
-import { addColumnSchema, backfillCompletedSchema, backfillIDSchema, backfillSchema, beginSchema, commitSchema, createIndexSchema, createTableSchema, dataChangeSchema, deleteSchema, dropColumnSchema, dropIndexSchema, dropTableSchema, identifierSchema, insertSchema, isDataChange, isSchemaChange, newRelationSchema, relationSchema, renameTableSchema, rollbackSchema, rowSchema, schemaChangeSchema, tableMetadataSchema, truncateSchema, updateColumnSchema, updateSchema, updateTableMetadataSchema } from "./current/data.js";
+import { addColumnSchema, backfillCompletedSchema, backfillIDSchema, backfillSchema, beginSchema, commitSchema, createIndexSchema, createTableSchema, dataChangeSchema, deleteSchema, downloadStatusSchema, dropColumnSchema, dropIndexSchema, dropTableSchema, identifierSchema, insertSchema, isDataChange, isSchemaChange, newRelationSchema, relationSchema, renameTableSchema, rollbackSchema, rowSchema, schemaChangeSchema, tableMetadataSchema, truncateSchema, updateColumnSchema, updateSchema, updateTableMetadataSchema } from "./current/data.js";
 import { changeStreamControlSchema, changeStreamDataSchema, changeStreamMessageSchema } from "./current/downstream.js";
 import { jsonObjectSchema, jsonValueSchema } from "./current/json.js";
 import { CHANGE_SOURCE_PATH } from "./current/path.js";
@@ -22,6 +22,7 @@ export {
   createTableSchema,
   dataChangeSchema,
   deleteSchema,
+  downloadStatusSchema,
   downstreamStatusMessageSchema,
   downstreamStatusSchema,
   dropColumnSchema,

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts

@@ -0,0 +1,100 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { WatermarkedChange } from './change-streamer-service.ts';
+import type { Subscriber } from './subscriber.ts';
+/**
+ * Initiates and tracks the progress of a change broadcasted to
+ * a set of subscribers.
+ *
+ * Creating a `Broadcast` automatically initiates the send.
+ *
+ * By default, {@link Broadcast.done} resolves when all subscribers
+ * have acked the change. However, {@link Broadcast.checkProgress()}
+ * can be called to resolve the broadcast earlier based on the flow
+ * control policy.
+ */
+export declare class Broadcast {
+    #private;
+    /**
+     * Sends the change to the subscribers without the tracking machinery.
+     * This is suitable for fire-and-forget (i.e. pipelined) sends.
+     */
+    static withoutTracking(subscribers: Iterable<Subscriber>, change: WatermarkedChange): void;
+    /**
+     * Broadcasts the `change` to the `subscribers` and tracks their
+     * completion.
+     */
+    constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange);
+    get isDone(): boolean;
+    get done(): Promise<void>;
+    /**
+     * Checks for pathological situations in which flow should be reenabled
+     * before all subscribers have acked.
+     *
+     * ### Background
+     *
+     * The purpose of flow control is to pull upstream replication changes
+     * no faster than the rate as they are processed by downstream subscribers
+     * in the steady state. In the change-streamer, this is done by occasionally
+     * waiting for ACKs from subscribers before continuing; without doing so,
+     * I/O buffers fill up and cause the system to spend most of its time in GC.
+     *
+     * However, the naive algorithm of always waiting for all subscribers (e.g.
+     * `Promise.all()`) can behave poorly in scenarios where subscribers
+     * are imbalanced:
+     * * New subscribers may have a backlog of changes to catch up with.
+     *   Having all subscribers wait for the new subscriber to catch up results
+     *   in delaying the entire application.
+     * * Broken TCP connections similarly require all subscribers to wait until
+     *   connection liveness checks kick in and disconnect the subscriber.
+     *
+     * A simplistic approach is to add a limit to the amount of time waiting for
+     * subscribers, i.e. an ack timeout. However, deciding what this timeout
+     * should be is non-trivial because of the heterogeneous nature of changes;
+     * while most changes operate on single rows and are relatively predictable
+     * in terms of running time, some changes are table-wide operations and can
+     * legitimately take an arbitrary amount of time. In such scenarios, a
+     * timeout that is too short can stop progress on replication altogether.
+     *
+     * ### Consensus-based Timeout Algorithm
+     *
+     * To address these shortcomings, a "consensus-based timeout" algorithm is
+     * used:
+     * * Wait for more than half of the subscribers to finish. (In
+     *   case of a single node, or the case of one replication-manager
+     *   and one view-syncer, this reduces to waiting for all subscribers.)
+     * * Once more than half of the subscribers have finished, proceed after
+     *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers
+     *   have finished.
+     *
+     * In other words, the subscribers themselves are used to determine the
+     * timeout of each batch of changes; the majority determines this when
+     * they complete, upon which a timeout is logically started.
+     *
+     * In the common case, the remaining subscribers finish soon afterward and
+     * the timeout never elapses. However, in pathological cases where a minority
+     * of subscribers have a disproportionate amount of load, some will still
+     * be processing (or otherwise unresponsive). These subscribers are given
+     * a bounded amount of time to catch up at each flushed batch, up to the
+     * timeout interval. This guarantees eventual catchup because the
+     * subscribers with a backlog of changes necessarily have a higher
+     * processing rate than the subscribers that finished (and are made to wait).
+     *
+     * ### Not implemented: Broken connection detection
+     *
+     * If a subscriber has not made progress for a certain interval, the
+     * algorithm could theoretically drop it preemptively, supplementing the
+     * existing websocket-level liveness checks.
+     *
+     * However, a more reliable approach would be to change the replicator
+     * to use non-blocking writes, and subsequently increase the frequency of
+     * connection-level liveness checks. The current synchronous replica writes
+     * can delay both ping responsiveness and change progress arbitrarily (e.g.
+     * a large index creation); an independently liveness check that is not
+     * delayed by synchronous writes on the subscriber would be a more failsafe
+     * solution.
+     *
+     * @returns `true` if the broadcast was already done or was marked done.
+     */
+    checkProgress(lc: LogContext, flowControlConsensusPaddingMs: number, now: number): boolean;
+}
+//# sourceMappingURL=broadcast.d.ts.map

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB;IAkB3B;;;OAGG;gBACS,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,iBAAiB;IAkCxE,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoEG;IACH,aAAa,CACX,EAAE,EAAE,UAAU,EACd,6BAA6B,EAAE,MAAM,EACrC,GAAG,EAAE,MAAM;CA+Cd"}

package/out/zero-cache/src/services/change-streamer/broadcast.js

@@ -0,0 +1,171 @@
+import { resolver } from "@rocicorp/resolver";
+class Broadcast {
+  /**
+   * Sends the change to the subscribers without the tracking machinery.
+   * This is suitable for fire-and-forget (i.e. pipelined) sends.
+   */
+  static withoutTracking(subscribers, change) {
+    for (const sub of subscribers) {
+      void sub.send(change);
+    }
+  }
+  #pending;
+  #completed;
+  #done = resolver();
+  #isDone = false;
+  #watermark;
+  #majority;
+  #start = performance.now();
+  #latestCompleted = Number.MAX_VALUE;
+  /**
+   * Broadcasts the `change` to the `subscribers` and tracks their
+   * completion.
+   */
+  constructor(subscribers, change) {
+    this.#pending = new Set(subscribers);
+    this.#completed = [];
+    this.#watermark = change[0];
+    this.#majority = Math.floor(this.#pending.size / 2) + 1;
+    for (const sub of this.#pending) {
+      const changes = sub.numPending + 1;
+      void sub.send(change).catch(() => {
+      }).finally(() => this.#markCompleted(sub, changes));
+    }
+    if (this.#pending.size === 0) {
+      this.#setDone();
+    }
+  }
+  #markCompleted(sub, changes) {
+    const elapsed = (this.#latestCompleted = performance.now()) - this.#start;
+    this.#completed.push({ sub, changes, elapsed });
+    this.#pending.delete(sub);
+    if (this.#pending.size === 0) {
+      this.#setDone();
+    }
+  }
+  #setDone() {
+    this.#isDone = true;
+    this.#done.resolve();
+  }
+  get isDone() {
+    return this.#isDone;
+  }
+  get done() {
+    return this.#done.promise;
+  }
+  /**
+   * Checks for pathological situations in which flow should be reenabled
+   * before all subscribers have acked.
+   *
+   * ### Background
+   *
+   * The purpose of flow control is to pull upstream replication changes
+   * no faster than the rate as they are processed by downstream subscribers
+   * in the steady state. In the change-streamer, this is done by occasionally
+   * waiting for ACKs from subscribers before continuing; without doing so,
+   * I/O buffers fill up and cause the system to spend most of its time in GC.
+   *
+   * However, the naive algorithm of always waiting for all subscribers (e.g.
+   * `Promise.all()`) can behave poorly in scenarios where subscribers
+   * are imbalanced:
+   * * New subscribers may have a backlog of changes to catch up with.
+   *   Having all subscribers wait for the new subscriber to catch up results
+   *   in delaying the entire application.
+   * * Broken TCP connections similarly require all subscribers to wait until
+   *   connection liveness checks kick in and disconnect the subscriber.
+   *
+   * A simplistic approach is to add a limit to the amount of time waiting for
+   * subscribers, i.e. an ack timeout. However, deciding what this timeout
+   * should be is non-trivial because of the heterogeneous nature of changes;
+   * while most changes operate on single rows and are relatively predictable
+   * in terms of running time, some changes are table-wide operations and can
+   * legitimately take an arbitrary amount of time. In such scenarios, a
+   * timeout that is too short can stop progress on replication altogether.
+   *
+   * ### Consensus-based Timeout Algorithm
+   *
+   * To address these shortcomings, a "consensus-based timeout" algorithm is
+   * used:
+   * * Wait for more than half of the subscribers to finish. (In
+   *   case of a single node, or the case of one replication-manager
+   *   and one view-syncer, this reduces to waiting for all subscribers.)
+   * * Once more than half of the subscribers have finished, proceed after
+   *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers
+   *   have finished.
+   *
+   * In other words, the subscribers themselves are used to determine the
+   * timeout of each batch of changes; the majority determines this when
+   * they complete, upon which a timeout is logically started.
+   *
+   * In the common case, the remaining subscribers finish soon afterward and
+   * the timeout never elapses. However, in pathological cases where a minority
+   * of subscribers have a disproportionate amount of load, some will still
+   * be processing (or otherwise unresponsive). These subscribers are given
+   * a bounded amount of time to catch up at each flushed batch, up to the
+   * timeout interval. This guarantees eventual catchup because the
+   * subscribers with a backlog of changes necessarily have a higher
+   * processing rate than the subscribers that finished (and are made to wait).
+   *
+   * ### Not implemented: Broken connection detection
+   *
+   * If a subscriber has not made progress for a certain interval, the
+   * algorithm could theoretically drop it preemptively, supplementing the
+   * existing websocket-level liveness checks.
+   *
+   * However, a more reliable approach would be to change the replicator
+   * to use non-blocking writes, and subsequently increase the frequency of
+   * connection-level liveness checks. The current synchronous replica writes
+   * can delay both ping responsiveness and change progress arbitrarily (e.g.
+   * a large index creation); an independently liveness check that is not
+   * delayed by synchronous writes on the subscriber would be a more failsafe
+   * solution.
+   *
+   * @returns `true` if the broadcast was already done or was marked done.
+   */
+  checkProgress(lc, flowControlConsensusPaddingMs, now) {
+    if (this.#pending.size === 0) {
+      return true;
+    }
+    const elapsed = now - this.#start;
+    if (this.#completed.length < this.#majority) {
+      if (elapsed >= 1e3) {
+        this.#logWithState(
+          lc,
+          `waiting for at least ${this.#majority} subscribers to finish`,
+          elapsed
+        );
+      }
+      return false;
+    }
+    if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {
+      this.#logWithState(
+        lc,
+        `continuing with ${this.#pending.size} subscriber(s) still pending`,
+        elapsed
+      );
+      this.#setDone();
+      return true;
+    }
+    return false;
+  }
+  #logWithState(lc, msg, elapsed) {
+    lc.withContext("watermark", this.#watermark).info?.(
+      `${msg} (${elapsed.toFixed(3)} ms)`,
+      {
+        completed: this.#completed.map((d) => ({
+          id: d.sub.id,
+          processed: d.changes,
+          elapsed: d.elapsed
+        })),
+        pending: [...this.#pending].map((sub) => ({
+          id: sub.id,
+          ...sub.getStats()
+        }))
+      }
+    );
+  }
+}
+export {
+  Broadcast
+};
+//# sourceMappingURL=broadcast.js.map

package/out/zero-cache/src/services/change-streamer/broadcast.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"broadcast.js","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers.\n *\n * Creating a `Broadcast` automatically initiates the send.\n *\n * By default, {@link Broadcast.done} resolves when all subscribers\n * have acked the change. However, {@link Broadcast.checkProgress()}\n * can be called to resolve the broadcast earlier based on the flow\n * control policy.\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n  ) {\n    for (const sub of subscribers) {\n      void sub.send(change);\n    }\n  }\n\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n  #latestCompleted = Number.MAX_VALUE;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange) {\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = change[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n\n    for (const sub of this.#pending) {\n      const changes = sub.numPending + 1; // add one for this `change`\n      void sub\n        .send(change)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, changes));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone();\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    const elapsed = (this.#latestCompleted = performance.now()) - this.#start;\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone();\n    }\n  }\n\n  #setDone() {\n    this.#isDone = true;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  /**\n   * Checks for pathological situations in which flow should be reenabled\n   * before all subscribers have acked.\n   *\n   * ### Background\n   *\n   * The purpose of flow control is to pull upstream replication changes\n   * no faster than the rate as they are processed by downstream subscribers\n   * in the steady state. In the change-streamer, this is done by occasionally\n   * waiting for ACKs from subscribers before continuing; without doing so,\n   * I/O buffers fill up and cause the system to spend most of its time in GC.\n   *\n   * However, the naive algorithm of always waiting for all subscribers (e.g.\n   * `Promise.all()`) can behave poorly in scenarios where subscribers\n   * are imbalanced:\n   * * New subscribers may have a backlog of changes to catch up with.\n   *   Having all subscribers wait for the new subscriber to catch up results\n   *   in delaying the entire application.\n   * * Broken TCP connections similarly require all subscribers to wait until\n   *   connection liveness checks kick in and disconnect the subscriber.\n   *\n   * A simplistic approach is to add a limit to the amount of time waiting for\n   * subscribers, i.e. an ack timeout. However, deciding what this timeout\n   * should be is non-trivial because of the heterogeneous nature of changes;\n   * while most changes operate on single rows and are relatively predictable\n   * in terms of running time, some changes are table-wide operations and can\n   * legitimately take an arbitrary amount of time. In such scenarios, a\n   * timeout that is too short can stop progress on replication altogether.\n   *\n   * ### Consensus-based Timeout Algorithm\n   *\n   * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n   * used:\n   * * Wait for more than half of the subscribers to finish. (In\n   *   case of a single node, or the case of one replication-manager\n   *   and one view-syncer, this reduces to waiting for all subscribers.)\n   * * Once more than half of the subscribers have finished, proceed after\n   *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers\n   *   have finished.\n   *\n   * In other words, the subscribers themselves are used to determine the\n   * timeout of each batch of changes; the majority determines this when\n   * they complete, upon which a timeout is logically started.\n   *\n   * In the common case, the remaining subscribers finish soon afterward and\n   * the timeout never elapses. However, in pathological cases where a minority\n   * of subscribers have a disproportionate amount of load, some will still\n   * be processing (or otherwise unresponsive). These subscribers are given\n   * a bounded amount of time to catch up at each flushed batch, up to the\n   * timeout interval. This guarantees eventual catchup because the\n   * subscribers with a backlog of changes necessarily have a higher\n   * processing rate than the subscribers that finished (and are made to wait).\n   *\n   * ### Not implemented: Broken connection detection\n   *\n   * If a subscriber has not made progress for a certain interval, the\n   * algorithm could theoretically drop it preemptively, supplementing the\n   * existing websocket-level liveness checks.\n   *\n   * However, a more reliable approach would be to change the replicator\n   * to use non-blocking writes, and subsequently increase the frequency of\n   * connection-level liveness checks. The current synchronous replica writes\n   * can delay both ping responsiveness and change progress arbitrarily (e.g.\n   * a large index creation); an independently liveness check that is not\n   * delayed by synchronous writes on the subscriber would be a more failsafe\n   * solution.\n   *\n   * @returns `true` if the broadcast was already done or was marked done.\n   */\n  checkProgress(\n    lc: LogContext,\n    flowControlConsensusPaddingMs: number,\n    now: number,\n  ) {\n    if (this.#pending.size === 0) {\n      return true;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          lc,\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n      return false;\n    }\n    // Note: In the implementation, #latestCompleted is always updated,\n    // even after the majority is reached. This is fine and does not affect\n    // the important properties of the algorithm.\n    if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {\n      this.#logWithState(\n        lc,\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n      this.#setDone();\n      return true;\n    }\n    return false;\n  }\n\n  #logWithState(lc: LogContext, msg: string, elapsed: number) {\n    lc.withContext('watermark', this.#watermark).info?.(\n      `${msg} (${elapsed.toFixed(3)} ms)`,\n      {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: [...this.#pending].map(sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      },\n    );\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n"],"names":[],"mappings":";AAgBO,MAAM,UAAU;AAAA;AAAA;AAAA;AAAA;AAAA,EAKrB,OAAO,gBACL,aACA,QACA;AACA,eAAW,OAAO,aAAa;AAC7B,WAAK,IAAI,KAAK,MAAM;AAAA,IACtB;AAAA,EACF;AAAA,EAES;AAAA,EACA;AAAA,EACA,QAAQ,SAAA;AAAA,EACjB,UAAU;AAAA,EAED;AAAA,EACA;AAAA,EAEA,SAAS,YAAY,IAAA;AAAA,EAC9B,mBAAmB,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA,EAM1B,YAAY,aAAmC,QAA2B;AACxE,SAAK,WAAW,IAAI,IAAI,WAAW;AACnC,SAAK,aAAa,CAAA;AAClB,SAAK,aAAa,OAAO,CAAC;AAC1B,SAAK,YAAY,KAAK,MAAM,KAAK,SAAS,OAAO,CAAC,IAAI;AAEtD,eAAW,OAAO,KAAK,UAAU;AAC/B,YAAM,UAAU,IAAI,aAAa;AACjC,WAAK,IACF,KAAK,MAAM,EACX,MAAM,MAAM;AAAA,MAAC,CAAC,EACd,QAAQ,MAAM,KAAK,eAAe,KAAK,OAAO,CAAC;AAAA,IACpD;AAGA,QAAI,KAAK,SAAS,SAAS,GAAG;AAC5B,WAAK,SAAA;AAAA,IACP;AAAA,EACF;AAAA,EAEA,eAAe,KAAiB,SAAiB;AAC/C,UAAM,WAAW,KAAK,mBAAmB,YAAY,IAAA,KAAS,KAAK;AACnE,SAAK,WAAW,KAAK,EAAC,KAAK,SAAS,SAAQ;AAC5C,SAAK,SAAS,OAAO,GAAG;AACxB,QAAI,KAAK,SAAS,SAAS,GAAG;AAC5B,WAAK,SAAA;AAAA,IACP;AAAA,EACF;AAAA,EAEA,WAAW;AACT,SAAK,UAAU;AACf,SAAK,MAAM,QAAA;AAAA,EACb;AAAA,EAEA,IAAI,SAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,OAAsB;AACxB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAuEA,cACE,IACA,+BACA,KACA;AACA,QAAI,KAAK,SAAS,SAAS,GAAG;AAC5B,aAAO;AAAA,IACT;AACA,UAAM,UAAU,MAAM,KAAK;AAC3B,QAAI,KAAK,WAAW,SAAS,KAAK,WAAW;AAC3C,UAAI,WAAW,KAAM;AACnB,aAAK;AAAA,UACH;AAAA,UACA,wBAAwB,KAAK,SAAS;AAAA,UACtC;AAAA,QAAA;AAAA,MAEJ;AACA,aAAO;AAAA,IACT;AAIA,QAAI,MAAM,KAAK,oBAAoB,+BAA+B;AAChE,WAAK;AAAA,QACH;AAAA,QACA,mBAAmB,KAAK,SAAS,IAAI;AAAA,QACrC;AAAA,MAAA;AAEF,WAAK,SAAA;AACL,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAAA,EAEA,cAAc,IAAgB,KAAa,SAAiB;AAC1D,OAAG,YAAY,aAAa,KAAK,UAAU,EAAE;AAAA,MAC3C,GAAG,GAAG,KAAK,QAAQ,QAAQ,CAAC,CAAC;AAAA,MAC7B;AAAA,QACE,WAAW,KAAK,WAAW,IAAI,CAAA,OAAM;AAAA,UACnC,IAAI,EAAE,IAAI;AAAA,UACV,WAAW,EAAE;AAAA,UACb,SAAS,EAAE;AAAA,QAAA,EACX;AAAA,QACF,SAAS,CAAC,GAAG,KAAK,QAAQ,EAAE,IAAI,CAAA,SAAQ;AAAA,UACtC,IAAI,IAAI;AAAA,UACR,GAAG,IAAI,SAAA;AAAA,QAAS,EAChB;AAAA,MAAA;AAAA,IACJ;AAAA,EAEJ;AACF;"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts

@@ -8,7 +8,7 @@ import { type ChangeStreamerService } from './change-streamer.ts';
 /**
  * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
  */
-export declare function initializeStreamer(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, changeDB: PostgresDB, changeSource: ChangeSource, subscriptionState: SubscriptionState, autoReset: boolean, backPressureLimitHeapProportion: number, setTimeoutFn?: typeof setTimeout): Promise<ChangeStreamerService>;
+export declare function initializeStreamer(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, changeDB: PostgresDB, changeSource: ChangeSource, subscriptionState: SubscriptionState, autoReset: boolean, backPressureLimitHeapProportion: number, flowControlConsensusPaddingSeconds: number, setTimeoutFn?: typeof setTimeout): Promise<ChangeStreamerService>;
 /**
  * Internally all Downstream messages (not just commits) are given a watermark.
  * These are used for internal ordering for:

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AAEzD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAY9B;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,OAAO,EAClB,+BAA+B,EAAE,MAAM,EACvC,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CAyBhC;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAYjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AAKzD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAY9B;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,OAAO,EAClB,+BAA+B,EAAE,MAAM,EACvC,kCAAkC,EAAE,MAAM,EAC1C,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA0BhC;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -1,11 +1,13 @@
 import { resolver } from "@rocicorp/resolver";
 import { getDefaultHighWaterMark } from "node:stream";
 import { unreachable } from "../../../../shared/src/asserts.js";
+import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { publishCriticalEvent } from "../../observability/events.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
 import { min } from "../../types/lexi-version.js";
 import { Subscription } from "../../types/subscription.js";
 import "../change-source/protocol/current/downstream.js";
-import { publishReplicationError } from "../replicator/replication-status.js";
+import { replicationStatusError, publishReplicationError } from "../replicator/replication-status.js";
 import { RunningState, UnrecoverableError, DEFAULT_MAX_RETRY_DELAY_MS } from "../running-state.js";
 import "./change-streamer.js";
 import { WrongReplicaVersion } from "./error-type-enum.js";
@@ -14,7 +16,7 @@ import { initChangeStreamerSchema } from "./schema/init.js";
 import { ensureReplicationConfig, markResetRequired, AutoResetSignal } from "./schema/tables.js";
 import { Storer } from "./storer.js";
 import { Subscriber } from "./subscriber.js";
-async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, subscriptionState, autoReset, backPressureLimitHeapProportion, setTimeoutFn = setTimeout) {
+async function initializeStreamer(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, changeSource, subscriptionState, autoReset, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds, setTimeoutFn = setTimeout) {
   await initChangeStreamerSchema(lc, changeDB, shard);
   await ensureReplicationConfig(
     lc,
@@ -35,6 +37,7 @@ async function initializeStreamer(lc, shard, taskID, discoveryAddress, discovery
     changeSource,
     autoReset,
     backPressureLimitHeapProportion,
+    flowControlConsensusPaddingSeconds,
     setTimeoutFn
   );
 }
@@ -65,7 +68,7 @@ class ChangeStreamerImpl {
     "Count of replicated transactions"
   );
   #stream;
-  constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, source, autoReset, backPressureLimitHeapProportion, setTimeoutFn = setTimeout) {
+  constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, source, autoReset, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds, setTimeoutFn = setTimeout) {
     this.id = `change-streamer`;
     this.#lc = lc.withContext("component", "change-streamer");
     this.#shard = shard;
@@ -84,12 +87,15 @@ class ChangeStreamerImpl {
       (err) => this.stop(err),
       backPressureLimitHeapProportion
     );
-    this.#forwarder = new Forwarder();
+    this.#forwarder = new Forwarder(lc, {
+      flowControlConsensusPaddingSeconds
+    });
     this.#autoReset = autoReset;
     this.#state = new RunningState(this.id, void 0, setTimeoutFn);
   }
   async run() {
     this.#lc.info?.("starting change stream");
+    this.#forwarder.startProgressMonitor();
     await this.#storer.assumeOwnership();
     const flushBytesThreshold = getDefaultHighWaterMark(false);
     while (this.#state.shouldRun()) {
@@ -137,10 +143,12 @@ class ChangeStreamerImpl {
               }
               break;
           }
-          unflushedBytes += this.#storer.store([watermark, change]);
-          const sent = this.#forwarder.forward([watermark, change]);
-          if (unflushedBytes >= flushBytesThreshold) {
-            await sent;
+          const entry = [watermark, change];
+          unflushedBytes += this.#storer.store(entry);
+          if (unflushedBytes < flushBytesThreshold) {
+            this.#forwarder.forward(entry);
+          } else {
+            await this.#forwarder.forwardWithFlowControl(entry);
             unflushedBytes = 0;
           }
           if (type === "commit" || type === "rollback") {
@@ -167,9 +175,14 @@ class ChangeStreamerImpl {
       }
       await Promise.all([
         this.#storer.stop(),
-        this.#state.backoff(this.#lc, err)
+        this.#state.backoff(this.#lc, err),
+        this.#state.retryDelay > 5e3 ? publishCriticalEvent(
+          this.#lc,
+          replicationStatusError(this.#lc, "Replicating", err)
+        ) : promiseVoid
       ]);
     }
+    this.#forwarder.stopProgressMonitor();
     this.#lc.info?.("ChangeStreamer stopped");
   }
   async #handleControlMessage(msg) {