@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.40

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts

@@ -1,11 +1,15 @@
 import type { Enum } from '../../../../shared/src/enum.ts';
 import * as v from '../../../../shared/src/valita.ts';
-import type { Sized, Source } from '../../types/streams.ts';
+import type { PreSerialized, Sized, Source } from '../../types/streams.ts';
+import type { Subscription } from '../../types/subscription.ts';
 import { type ChangeStreamData } from '../change-source/protocol/current/downstream.ts';
+import type { BackfillRequest } from '../change-source/protocol/current/upstream.ts';
 import type { ReplicatorMode } from '../replicator/replicator.ts';
 import type { Service } from '../service.ts';
+import type { PreSerializedBatch } from './broadcast.ts';
 import * as ErrorType from './error-type-enum.ts';
-import type { SnapshotMessage } from './snapshot.ts';
+import type { SnapshotMessage } from './snapshot-message.ts';
+import type { LocalReplica, SubscribeDownstream } from './subscribe.ts';
 type ErrorType = Enum<typeof ErrorType>;
 export type ChangeTag = ChangeStreamData[1]['tag'];
 /**
@@ -62,9 +66,9 @@ export interface ChangeStreamer {
      * date. Each result contains parsed, validated data and its approximate
      * serialized transport size.
      */
-    subscribe(ctx: SubscriberContext): Promise<Source<SizedDownstream>>;
+    subscribe(ctx: SubscriberContext): Promise<Source<Sized<SubscribeDownstream>>>;
 }
-export declare const PROTOCOL_VERSION = 6;
+export declare const PROTOCOL_VERSION = 8;
 export type SubscriberContext = {
     /**
      * The supported change-streamer protocol version.
@@ -97,18 +101,29 @@ export type SubscriberContext = {
      */
     watermark: string;
     /**
-     * Whether this is the first subscription request made by the task,
-     * i.e. indicating that the watermark comes from a restored replica
-     * backup. The ChangeStreamer uses this to determine which changes
-     * are safe to purge from the Storer.
+     * Whether the subscriber supports batched WebSocket frames.
      */
-    initial: boolean;
+    wsBatched?: boolean | undefined;
     /**
-     * Legacy topology hint retained on the wire. It no longer affects local
-     * routing: the change-streamer writes the SQLite log itself, so no
-     * subscriber's ACK advances its head or releases its catchup barrier.
+     * The subscriber's pending backfills as of its `watermark`. Reported by
+     * subscribers as of protocol v8. `undefined` for older subscribers.
      */
-    logsChangeStream: boolean;
+    backfills?: BackfillRequest[] | undefined;
+    /**
+     * Whether the subscriber understands the `draining` status, i.e. it will
+     * switch over to another replication-manager when the current one shuts
+     * down. A draining replication-manager waits for drainable subscribers to
+     * leave before exiting.
+     */
+    drainable?: boolean | undefined;
+    /**
+     * Whether the subscriber is switching over from another subscription with
+     * a replica that it is currently serving. If the replica version is
+     * incompatible, such a subscriber is held (see {@link Status.restore})
+     * rather than being immediately rejected. Subscribers that are booting up
+     * never set this.
+     */
+    switchover?: boolean | undefined;
 };
 /**
  * The StatusMessage payload for now is empty, but can be extended to
@@ -124,6 +139,33 @@ export declare const statusSchema: v.ObjectType<{
         } | undefined;
         nextSendTimeMs: number;
     }>;
+    /**
+     * Sent by a replication-manager that is shutting down to its (drainable)
+     * serving subscribers. The subscriber should switch over to another
+     * replication-manager (see {@link SubscriberContext.switchover}) while
+     * continuing to consume this subscription until the switchover succeeds.
+     */
+    draining: v.Optional<boolean>;
+    /**
+     * Sent on a {@link SubscriberContext.switchover switchover} subscription
+     * to a replication-manager running an incompatible replica version.
+     * Rather than being immediately rejected with
+     * {@link ErrorType.WrongReplicaVersion}, the subscriber is held until it
+     * is granted permission to restore the replication-manager's backup,
+     * which limits the number of subscribers that are restoring (and thus not
+     * serving) at the same time.
+     *
+     * * `queued`: the subscriber is waiting for permission to restore. It
+     *   should continue consuming its existing subscription (if any).
+     * * `granted`: the subscriber should (gracefully) restart to restore a
+     *   new replica.
+     */
+    restore: v.Optional<{
+        state: "queued";
+        position: number;
+    } | {
+        state: "granted";
+    }>;
 }, undefined>;
 export type Status = v.Infer<typeof statusSchema>;
 export declare const statusMessageSchema: v.TupleType<[v.Type<"status">, v.ObjectType<{
@@ -136,6 +178,33 @@ export declare const statusMessageSchema: v.TupleType<[v.Type<"status">, v.Objec
         } | undefined;
         nextSendTimeMs: number;
     }>;
+    /**
+     * Sent by a replication-manager that is shutting down to its (drainable)
+     * serving subscribers. The subscriber should switch over to another
+     * replication-manager (see {@link SubscriberContext.switchover}) while
+     * continuing to consume this subscription until the switchover succeeds.
+     */
+    draining: v.Optional<boolean>;
+    /**
+     * Sent on a {@link SubscriberContext.switchover switchover} subscription
+     * to a replication-manager running an incompatible replica version.
+     * Rather than being immediately rejected with
+     * {@link ErrorType.WrongReplicaVersion}, the subscriber is held until it
+     * is granted permission to restore the replication-manager's backup,
+     * which limits the number of subscribers that are restoring (and thus not
+     * serving) at the same time.
+     *
+     * * `queued`: the subscriber is waiting for permission to restore. It
+     *   should continue consuming its existing subscription (if any).
+     * * `granted`: the subscriber should (gracefully) restart to restore a
+     *   new replica.
+     */
+    restore: v.Optional<{
+        state: "queued";
+        position: number;
+    } | {
+        state: "granted";
+    }>;
 }, undefined>]>;
 /**
  * A StatusMessage will be immediately sent on a (v2+) subscription to
@@ -147,11 +216,23 @@ export type StatusMessage = v.Infer<typeof statusMessageSchema>;
 declare const subscriptionErrorSchema: v.ObjectType<{
     type: v.Type<number>;
     message: v.Optional<string>;
+    /**
+     * The replica version of the replication-manager, sent with
+     * {@link ErrorType.WrongReplicaVersion} so that a subscriber with a newer
+     * replica can avoid restoring an older one.
+     */
+    replicaVersion: v.Optional<string>;
 }, undefined>;
 export type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;
 declare const errorSchema: v.TupleType<[v.Type<"error">, v.ObjectType<{
     type: v.Type<number>;
     message: v.Optional<string>;
+    /**
+     * The replica version of the replication-manager, sent with
+     * {@link ErrorType.WrongReplicaVersion} so that a subscriber with a newer
+     * replica can avoid restoring an older one.
+     */
+    replicaVersion: v.Optional<string>;
 }, undefined>]>;
 export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status">, v.ObjectType<{
     tag: v.Type<"status">;
@@ -163,6 +244,33 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
         } | undefined;
         nextSendTimeMs: number;
     }>;
+    /**
+     * Sent by a replication-manager that is shutting down to its (drainable)
+     * serving subscribers. The subscriber should switch over to another
+     * replication-manager (see {@link SubscriberContext.switchover}) while
+     * continuing to consume this subscription until the switchover succeeds.
+     */
+    draining: v.Optional<boolean>;
+    /**
+     * Sent on a {@link SubscriberContext.switchover switchover} subscription
+     * to a replication-manager running an incompatible replica version.
+     * Rather than being immediately rejected with
+     * {@link ErrorType.WrongReplicaVersion}, the subscriber is held until it
+     * is granted permission to restore the replication-manager's backup,
+     * which limits the number of subscribers that are restoring (and thus not
+     * serving) at the same time.
+     *
+     * * `queued`: the subscriber is waiting for permission to restore. It
+     *   should continue consuming its existing subscription (if any).
+     * * `granted`: the subscriber should (gracefully) restart to restore a
+     *   new replica.
+     */
+    restore: v.Optional<{
+        state: "queued";
+        position: number;
+    } | {
+        state: "granted";
+    }>;
 }, undefined>]>, v.UnionType<[v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
     json: v.Optional<"p" | "s">;
@@ -194,7 +302,7 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
             type?: "default" | "full" | "index" | "nothing" | undefined;
         };
     }>;
-    key: v.Type<Record<string, import("../../../../shared/src/bigint-json.ts").JSONValue> | null>;
+    key: v.UnionType<[v.Type<null>, v.Type<Record<string, import("../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     new: v.Type<Record<string, import("../../../../shared/src/bigint-json.ts").JSONValue>>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"delete">;
@@ -234,6 +342,18 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
     rowValues: v.ArrayType<v.ArrayType<v.Type<import("../../../../shared/src/bigint-json.ts").JSONValue>>>;
+    progressMarks: v.Optional<{
+        previous?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+        current: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        };
+    }>;
     status: v.Optional<{
         rows: number;
         totalRows: number;
@@ -378,6 +498,13 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
     }, undefined>;
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
+    progressMarks: v.Optional<{
+        previous?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+    }>;
     status: v.Optional<{
         rows: number;
         totalRows: number;
@@ -393,6 +520,12 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
 }, undefined>]>]>, v.TupleType<[v.Type<"error">, v.ObjectType<{
     type: v.Type<number>;
     message: v.Optional<string>;
+    /**
+     * The replica version of the replication-manager, sent with
+     * {@link ErrorType.WrongReplicaVersion} so that a subscriber with a newer
+     * replica can avoid restoring an older one.
+     */
+    replicaVersion: v.Optional<string>;
 }, undefined>]>]>;
 export type Error = v.Infer<typeof errorSchema>;
 export declare function errorTypeToReadableName(val: ErrorType): "Unknown" | "WatermarkTooOld" | "WrongReplicaVersion";
@@ -413,9 +546,12 @@ export type SizedDownstream = Sized<Downstream>;
 export interface ChangeStreamerService extends Omit<ChangeStreamer, 'subscribe'>, Service {
     /**
      * The server-side interface overrides `subscribe()` to return a stream
-     * of already-stringified {@link Downstream} payloads.
+     * of already-stringified {@link Downstream} payloads or pre-serialized Buffers.
+     *
+     * If a `downstream` subscription is supplied it is used directly as the
+     * returned Source. Otherwise, a new one is created.
      */
-    subscribe(ctx: SubscriberContext): Promise<Source<string>>;
+    subscribe(ctx: SubscriberContext, downstream?: Subscription<string | PreSerializedBatch>): Promise<Source<string | PreSerialized>>;
     /**
      * Starts a snapshot reservation to preserve change-log entries while
      * a soon-to-be-subscriber downloads the current replica backup. Once
@@ -425,7 +561,15 @@ export interface ChangeStreamerService extends Omit<ChangeStreamer, 'subscribe'>
      * until the client with that `taskID` subscribes, or if the snapshot
      * connection terminates.
      */
-    startSnapshotReservation(taskID: string): Promise<Source<SnapshotMessage>>;
+    startSnapshotReservation(taskID: string, opts?: {
+        localReplica?: LocalReplica | undefined;
+    }): Promise<Source<SnapshotMessage>>;
+    /**
+     * Signals (drainable) serving subscribers to switch over to another
+     * replication-manager, and resolves when they have all disconnected.
+     * Subscribers continue to be served in the meantime.
+     */
+    drain?(): Promise<void>;
     /**
      * Informs the change-streamer of the watermark up to which the replica has
      * been backed up. This serves two purposes:

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AACzD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,KAAK,EAAE,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAClD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;GAOG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;;OAKG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;CACrE;AAgCD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;;;OAIG;IACH,gBAAgB,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;aASvB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,SAAS,yDAWrD;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,0EAA0E;AAC1E,MAAM,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;AAEhD,MAAM,WAAW,qBACf,SAAQ,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,EAAE,OAAO;IAClD;;;OAGG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAE3D;;;;;;;;OAQG;IACH,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;IAE3E;;;;;;;;;OASG;IACH,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/C"}
+{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AACzD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,aAAa,EAAE,KAAK,EAAE,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACzE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AACnF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAClD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAC,YAAY,EAAE,mBAAmB,EAAC,MAAM,gBAAgB,CAAC;AAEtE,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;GAOG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;;OAKG;IACH,SAAS,CACP,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;CAChD;AAoDD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAEhC;;;OAGG;IACH,SAAS,CAAC,EAAE,eAAe,EAAE,GAAG,SAAS,CAAC;IAE1C;;;;;OAKG;IACH,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAEhC;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;IAUvB;;;;;OAKG;;IAGH;;;;;;;;;;;;;OAaG;;;;;;;aAOH,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;IAhC9B;;;;;OAKG;;IAGH;;;;;;;;;;;;;OAaG;;;;;;;eAW0E,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;IAG3B;;;;OAIG;;aAEH,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;IAVf;;;;OAIG;;eAMqE,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;IAzD3B;;;;;OAKG;;IAGH;;;;;;;;;;;;;OAaG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAwBH;;;;OAIG;;iBAYJ,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,SAAS,yDAWrD;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,0EAA0E;AAC1E,MAAM,MAAM,eAAe,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;AAEhD,MAAM,WAAW,qBACf,SAAQ,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,EAAE,OAAO;IAClD;;;;;;OAMG;IACH,SAAS,CACP,GAAG,EAAE,iBAAiB,EACtB,UAAU,CAAC,EAAE,YAAY,CAAC,MAAM,GAAG,kBAAkB,CAAC,GACrD,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC;IAE3C;;;;;;;;OAQG;IACH,wBAAwB,CACtB,MAAM,EAAE,MAAM,EACd,IAAI,CAAC,EAAE;QAAC,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAA;KAAC,GAC/C,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;IAEpC;;;;OAIG;IACH,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAExB;;;;;;;;;OASG;IACH,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/C"}

package/out/zero-cache/src/services/change-streamer/change-streamer.js

@@ -11,12 +11,43 @@ var statusSchema = valita_exports.object({
 	lagReport: valita_exports.object({
 		lastTimings: changeSourceTimingsSchema.optional(),
 		nextSendTimeMs: valita_exports.number()
-	}).optional()
+	}).optional(),
+	/**
+	* Sent by a replication-manager that is shutting down to its (drainable)
+	* serving subscribers. The subscriber should switch over to another
+	* replication-manager (see {@link SubscriberContext.switchover}) while
+	* continuing to consume this subscription until the switchover succeeds.
+	*/
+	draining: valita_exports.boolean().optional(),
+	/**
+	* Sent on a {@link SubscriberContext.switchover switchover} subscription
+	* to a replication-manager running an incompatible replica version.
+	* Rather than being immediately rejected with
+	* {@link ErrorType.WrongReplicaVersion}, the subscriber is held until it
+	* is granted permission to restore the replication-manager's backup,
+	* which limits the number of subscribers that are restoring (and thus not
+	* serving) at the same time.
+	*
+	* * `queued`: the subscriber is waiting for permission to restore. It
+	*   should continue consuming its existing subscription (if any).
+	* * `granted`: the subscriber should (gracefully) restart to restore a
+	*   new replica.
+	*/
+	restore: valita_exports.union(valita_exports.object({
+		state: valita_exports.literal("queued"),
+		position: valita_exports.number()
+	}), valita_exports.object({ state: valita_exports.literal("granted") })).optional()
 });
 var statusMessageSchema = valita_exports.tuple([valita_exports.literal("status"), statusSchema]);
 var subscriptionErrorSchema = valita_exports.object({
 	type: valita_exports.number(),
-	message: valita_exports.string().optional()
+	message: valita_exports.string().optional(),
+	/**
+	* The replica version of the replication-manager, sent with
+	* {@link ErrorType.WrongReplicaVersion} so that a subscriber with a newer
+	* replica can avoid restoring an older one.
+	*/
+	replicaVersion: valita_exports.string().optional()
 });
 var errorSchema = valita_exports.tuple([valita_exports.literal("error"), subscriptionErrorSchema]);
 var downstreamSchema = valita_exports.union(statusMessageSchema, changeStreamDataSchema, errorSchema);

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Sized, Source} from '../../types/streams.ts';\nimport {\n  changeStreamDataSchema,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering while replaying stored changes and\n * filtering changes already seen by a subscriber.\n *\n * Only commit watermarks are exposed to subscribers. The JSON string is the\n * canonical serialization shared by storage and live forwarding.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date. Each result contains parsed, validated data and its approximate\n   * serialized transport size.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<SizedDownstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n//   - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n//     the upstream commit timestamp for end-to-end serving lag measurement.\n//     The stream is parsed in 'passthrough' mode, so an older peer ignores the\n//     field, and a newer peer treats its absence (including in changes\n//     replayed from the Change DB) as \"no commit time reported\".\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string;\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n\n  /**\n   * Legacy topology hint retained on the wire. It no longer affects local\n   * routing: the change-streamer writes the SQLite log itself, so no\n   * subscriber's ACK advances its head or releases its catchup barrier.\n   */\n  logsChangeStream: boolean;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\n/** A downstream message and its approximate serialized transport size. */\nexport type SizedDownstream = Sized<Downstream>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<string>>;\n\n  /**\n   * Starts a snapshot reservation to preserve change-log entries while\n   * a soon-to-be-subscriber downloads the current replica backup. Once\n   * the change-streamer knows that a backup is available\n   * (via {@link trackBackupWatermark}), it sends a confirmation\n   * SnapshotMessage and reserves the corresponding entries in the change-log\n   * until the client with that `taskID` subscribes, or if the snapshot\n   * connection terminates.\n   */\n  startSnapshotReservation(taskID: string): Promise<Source<SnapshotMessage>>;\n\n  /**\n   * Informs the change-streamer of the watermark up to which the replica has\n   * been backed up. This serves two purposes:\n   * - It serves as the maximum watermark up to which the change-log can be\n   *   purged. Note that the change-streamer also takes into account the\n   *   positions of its subscribers and snapshot reservations when purging\n   *   changes in the change-log.\n   * - In RMv2, this ACK's the upstream change-source to allow its buffer of\n   *   changes (e.g. the replication_slot) to advance.\n   */\n  trackBackupWatermark(watermark: string): void;\n}\n"],"mappings":";;;;;;;;AAoKA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {PreSerialized, Sized, Source} from '../../types/streams.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n  changeStreamDataSchema,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {LocalReplica, SubscribeDownstream} from './subscribe.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering while replaying stored changes and\n * filtering changes already seen by a subscriber.\n *\n * Only commit watermarks are exposed to subscribers. The JSON string is the\n * canonical serialization shared by storage and live forwarding.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date. Each result contains parsed, validated data and its approximate\n   * serialized transport size.\n   */\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n//   - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n//     the upstream commit timestamp for end-to-end serving lag measurement.\n//     The stream is parsed in 'passthrough' mode, so an older peer ignores the\n//     field, and a newer peer treats its absence (including in changes\n//     replayed from the Change DB) as \"no commit time reported\".\n// v7: v1.12\n//   - Merges the separate `/snapshot` and `/changes` connections into a single\n//     bidirectional `/subscribe` connection (see subscribe.ts). The view-syncer\n//     drives the connection with upstream `reserve-snapshot` / `start-subscription`\n//     application messages; the change-streamer replies with a `reserved`\n//     message and then the change stream on the same socket. This guarantees the\n//     replication-manager that reserves the change log is the one that serves the\n//     subscription. The legacy `/snapshot` + `/changes` endpoints remain for\n//     v4-v6 subscribers during rollout.\n//   - Drops the legacy \"initial\" and \"logsChangeStream\" query parameters from the\n//     subscriber context\n// v8:\n//   - Adds `backfills` to the `start-subscription` context, with which the\n//     subscriber reports its pending backfills (and their progress).\n//   - Adds `progressMarks` to `backfill` and `backfill-completed` messages,\n//     which subscribers use to ignore backfill data that would otherwise\n//     create a gap (see `acceptBackfill()`).\n//   Both fields are optional in the schemas so that v7 peers (and changes\n//   persisted in the change-log by v7 replication-managers) remain compatible,\n//   but they should be considered non-optional as of v8.\n\nexport const PROTOCOL_VERSION = 8;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string;\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether the subscriber supports batched WebSocket frames.\n   */\n  wsBatched?: boolean | undefined;\n\n  /**\n   * The subscriber's pending backfills as of its `watermark`. Reported by\n   * subscribers as of protocol v8. `undefined` for older subscribers.\n   */\n  backfills?: BackfillRequest[] | undefined;\n\n  /**\n   * Whether the subscriber understands the `draining` status, i.e. it will\n   * switch over to another replication-manager when the current one shuts\n   * down. A draining replication-manager waits for drainable subscribers to\n   * leave before exiting.\n   */\n  drainable?: boolean | undefined;\n\n  /**\n   * Whether the subscriber is switching over from another subscription with\n   * a replica that it is currently serving. If the replica version is\n   * incompatible, such a subscriber is held (see {@link Status.restore})\n   * rather than being immediately rejected. Subscribers that are booting up\n   * never set this.\n   */\n  switchover?: boolean | undefined;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n\n  /**\n   * Sent by a replication-manager that is shutting down to its (drainable)\n   * serving subscribers. The subscriber should switch over to another\n   * replication-manager (see {@link SubscriberContext.switchover}) while\n   * continuing to consume this subscription until the switchover succeeds.\n   */\n  draining: v.boolean().optional(),\n\n  /**\n   * Sent on a {@link SubscriberContext.switchover switchover} subscription\n   * to a replication-manager running an incompatible replica version.\n   * Rather than being immediately rejected with\n   * {@link ErrorType.WrongReplicaVersion}, the subscriber is held until it\n   * is granted permission to restore the replication-manager's backup,\n   * which limits the number of subscribers that are restoring (and thus not\n   * serving) at the same time.\n   *\n   * * `queued`: the subscriber is waiting for permission to restore. It\n   *   should continue consuming its existing subscription (if any).\n   * * `granted`: the subscriber should (gracefully) restart to restore a\n   *   new replica.\n   */\n  restore: v\n    .union(\n      v.object({state: v.literal('queued'), position: v.number()}),\n      v.object({state: v.literal('granted')}),\n    )\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n  /**\n   * The replica version of the replication-manager, sent with\n   * {@link ErrorType.WrongReplicaVersion} so that a subscriber with a newer\n   * replica can avoid restoring an older one.\n   */\n  replicaVersion: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\n/** A downstream message and its approximate serialized transport size. */\nexport type SizedDownstream = Sized<Downstream>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads or pre-serialized Buffers.\n   *\n   * If a `downstream` subscription is supplied it is used directly as the\n   * returned Source. Otherwise, a new one is created.\n   */\n  subscribe(\n    ctx: SubscriberContext,\n    downstream?: Subscription<string | PreSerializedBatch>,\n  ): Promise<Source<string | PreSerialized>>;\n\n  /**\n   * Starts a snapshot reservation to preserve change-log entries while\n   * a soon-to-be-subscriber downloads the current replica backup. Once\n   * the change-streamer knows that a backup is available\n   * (via {@link trackBackupWatermark}), it sends a confirmation\n   * SnapshotMessage and reserves the corresponding entries in the change-log\n   * until the client with that `taskID` subscribes, or if the snapshot\n   * connection terminates.\n   */\n  startSnapshotReservation(\n    taskID: string,\n    opts?: {localReplica?: LocalReplica | undefined},\n  ): Promise<Source<SnapshotMessage>>;\n\n  /**\n   * Signals (drainable) serving subscribers to switch over to another\n   * replication-manager, and resolves when they have all disconnected.\n   * Subscribers continue to be served in the meantime.\n   */\n  drain?(): Promise<void>;\n\n  /**\n   * Informs the change-streamer of the watermark up to which the replica has\n   * been backed up. This serves two purposes:\n   * - It serves as the maximum watermark up to which the change-log can be\n   *   purged. Note that the change-streamer also takes into account the\n   *   positions of its subscribers and snapshot reservations when purging\n   *   changes in the change-log.\n   * - In RMv2, this ACK's the upstream change-source to allow its buffer of\n   *   changes (e.g. the replication_slot) to advance.\n   */\n  trackBackupWatermark(watermark: string): void;\n}\n"],"mappings":";;;;;;;;AA2MA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;;;;;;;CAQZ,UAAU,eAAE,QAAQ,EAAE,SAAS;;;;;;;;;;;;;;;CAgB/B,SAAS,eACN,MACC,eAAE,OAAO;EAAC,OAAO,eAAE,QAAQ,QAAQ;EAAG,UAAU,eAAE,OAAO;CAAC,CAAC,GAC3D,eAAE,OAAO,EAAC,OAAO,eAAE,QAAQ,SAAS,EAAC,CAAC,CACxC,EACC,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;;;;;;CAM7B,gBAAgB,eAAE,OAAO,EAAE,SAAS;AACtC,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}

package/out/zero-cache/src/services/change-streamer/drain-tracker.d.ts

@@ -0,0 +1,24 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Subscription } from '../../types/subscription.ts';
+import type { PreSerializedBatch } from './broadcast.ts';
+import type { Subscriber } from './subscriber.ts';
+/**
+ * Tracks the {@link Subscriber.drainable drainable} subscribers of the
+ * change-streamer, i.e. those that switch over to another
+ * replication-manager when this one drains.
+ *
+ * Upon {@link drain}, drainable subscribers are sent a `draining` status
+ * (as are drainable subscribers that subscribe while draining), and the
+ * returned promise resolves once they have all disconnected. Subscribers
+ * continue to be served in the meantime.
+ */
+export declare class DrainTracker {
+    #private;
+    constructor(lc: LogContext, setIntervalFn?: typeof setInterval, clearIntervalFn?: typeof clearInterval);
+    get draining(): boolean;
+    register(subscriber: Subscriber, downstream: Subscription<string | PreSerializedBatch>): void;
+    drain(): Promise<void>;
+    /** Resolves any pending drain, e.g. when the change-streamer stops. */
+    stop(): void;
+}
+//# sourceMappingURL=drain-tracker.d.ts.map

package/out/zero-cache/src/services/change-streamer/drain-tracker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"drain-tracker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/drain-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAIhD;;;;;;;;;GASG;AACH,qBAAa,YAAY;;IAOvB,YACE,EAAE,EAAE,UAAU,EAEd,aAAa,GAAE,OAAO,WAAyB,EAC/C,eAAe,GAAE,OAAO,aAA6B,EAKtD;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,QAAQ,CACN,UAAU,EAAE,UAAU,EACtB,UAAU,EAAE,YAAY,CAAC,MAAM,GAAG,kBAAkB,CAAC,QActD;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAuBrB;IAED,uEAAuE;IACvE,IAAI,SAEH;CAQF"}

package/out/zero-cache/src/services/change-streamer/drain-tracker.js

@@ -0,0 +1,67 @@
+import { resolver } from "@rocicorp/resolver";
+//#region ../zero-cache/src/services/change-streamer/drain-tracker.ts
+var PROGRESS_LOG_INTERVAL_MS = 3e4;
+/**
+* Tracks the {@link Subscriber.drainable drainable} subscribers of the
+* change-streamer, i.e. those that switch over to another
+* replication-manager when this one drains.
+*
+* Upon {@link drain}, drainable subscribers are sent a `draining` status
+* (as are drainable subscribers that subscribe while draining), and the
+* returned promise resolves once they have all disconnected. Subscribers
+* continue to be served in the meantime.
+*/
+var DrainTracker = class {
+	#lc;
+	#subscribers = /* @__PURE__ */ new Set();
+	#setInterval;
+	#clearInterval;
+	#draining;
+	constructor(lc, setIntervalFn = setInterval, clearIntervalFn = clearInterval) {
+		this.#lc = lc.withContext("component", "drain-tracker");
+		this.#setInterval = setIntervalFn;
+		this.#clearInterval = clearIntervalFn;
+	}
+	get draining() {
+		return this.#draining !== void 0;
+	}
+	register(subscriber, downstream) {
+		if (!subscriber.drainable) return;
+		this.#subscribers.add(subscriber);
+		if (this.#draining) subscriber.sendControlStatus({
+			tag: "status",
+			draining: true
+		});
+		downstream.addCloseHandler(() => {
+			if (this.#subscribers.delete(subscriber)) this.#checkDrained();
+		});
+	}
+	drain() {
+		if (!this.#draining) {
+			const draining = this.#draining = resolver();
+			this.#lc.info?.(`draining: signaling ${this.#subscribers.size} subscribers to switch over`);
+			for (const sub of this.#subscribers) sub.sendControlStatus({
+				tag: "status",
+				draining: true
+			});
+			const progress = this.#setInterval(() => this.#lc.info?.(`draining: waiting for ${this.#subscribers.size} subscribers to switch over`, Array.from(this.#subscribers, (sub) => sub.id)), PROGRESS_LOG_INTERVAL_MS);
+			draining.promise.finally(() => this.#clearInterval(progress));
+			this.#checkDrained();
+		}
+		return this.#draining.promise;
+	}
+	/** Resolves any pending drain, e.g. when the change-streamer stops. */
+	stop() {
+		this.#draining?.resolve();
+	}
+	#checkDrained() {
+		if (this.#draining && this.#subscribers.size === 0) {
+			this.#lc.info?.(`drained all subscribers`);
+			this.#draining.resolve();
+		}
+	}
+};
+//#endregion
+export { DrainTracker };
+
+//# sourceMappingURL=drain-tracker.js.map

package/out/zero-cache/src/services/change-streamer/drain-tracker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"drain-tracker.js","names":["#lc","#subscribers","#setInterval","#clearInterval","#draining","#checkDrained"],"sources":["../../../../../../zero-cache/src/services/change-streamer/drain-tracker.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nconst PROGRESS_LOG_INTERVAL_MS = 30_000;\n\n/**\n * Tracks the {@link Subscriber.drainable drainable} subscribers of the\n * change-streamer, i.e. those that switch over to another\n * replication-manager when this one drains.\n *\n * Upon {@link drain}, drainable subscribers are sent a `draining` status\n * (as are drainable subscribers that subscribe while draining), and the\n * returned promise resolves once they have all disconnected. Subscribers\n * continue to be served in the meantime.\n */\nexport class DrainTracker {\n  readonly #lc: LogContext;\n  readonly #subscribers = new Set<Subscriber>();\n  readonly #setInterval: typeof setInterval;\n  readonly #clearInterval: typeof clearInterval;\n  #draining: Resolver<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    // Injectable timers, for deterministic testing.\n    setIntervalFn: typeof setInterval = setInterval,\n    clearIntervalFn: typeof clearInterval = clearInterval,\n  ) {\n    this.#lc = lc.withContext('component', 'drain-tracker');\n    this.#setInterval = setIntervalFn;\n    this.#clearInterval = clearIntervalFn;\n  }\n\n  get draining(): boolean {\n    return this.#draining !== undefined;\n  }\n\n  register(\n    subscriber: Subscriber,\n    downstream: Subscription<string | PreSerializedBatch>,\n  ) {\n    if (!subscriber.drainable) {\n      return;\n    }\n    this.#subscribers.add(subscriber);\n    if (this.#draining) {\n      subscriber.sendControlStatus({tag: 'status', draining: true});\n    }\n    downstream.addCloseHandler(() => {\n      if (this.#subscribers.delete(subscriber)) {\n        this.#checkDrained();\n      }\n    });\n  }\n\n  drain(): Promise<void> {\n    if (!this.#draining) {\n      const draining = (this.#draining = resolver<void>());\n      this.#lc.info?.(\n        `draining: signaling ${this.#subscribers.size} subscribers ` +\n          `to switch over`,\n      );\n      for (const sub of this.#subscribers) {\n        sub.sendControlStatus({tag: 'status', draining: true});\n      }\n      const progress = this.#setInterval(\n        () =>\n          this.#lc.info?.(\n            `draining: waiting for ${this.#subscribers.size} ` +\n              `subscribers to switch over`,\n            Array.from(this.#subscribers, sub => sub.id),\n          ),\n        PROGRESS_LOG_INTERVAL_MS,\n      );\n      void draining.promise.finally(() => this.#clearInterval(progress));\n      this.#checkDrained();\n    }\n    return this.#draining.promise;\n  }\n\n  /** Resolves any pending drain, e.g. when the change-streamer stops. */\n  stop() {\n    this.#draining?.resolve();\n  }\n\n  #checkDrained() {\n    if (this.#draining && this.#subscribers.size === 0) {\n      this.#lc.info?.(`drained all subscribers`);\n      this.#draining.resolve();\n    }\n  }\n}\n"],"mappings":";;AAMA,IAAM,2BAA2B;;;;;;;;;;;AAYjC,IAAa,eAAb,MAA0B;CACxB;CACA,+BAAwB,IAAI,IAAgB;CAC5C;CACA;CACA;CAEA,YACE,IAEA,gBAAoC,aACpC,kBAAwC,eACxC;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKE,eAAe;EACpB,KAAKC,iBAAiB;CACxB;CAEA,IAAI,WAAoB;EACtB,OAAO,KAAKC,cAAc,KAAA;CAC5B;CAEA,SACE,YACA,YACA;EACA,IAAI,CAAC,WAAW,WACd;EAEF,KAAKH,aAAa,IAAI,UAAU;EAChC,IAAI,KAAKG,WACP,WAAW,kBAAkB;GAAC,KAAK;GAAU,UAAU;EAAI,CAAC;EAE9D,WAAW,sBAAsB;GAC/B,IAAI,KAAKH,aAAa,OAAO,UAAU,GACrC,KAAKI,cAAc;EAEvB,CAAC;CACH;CAEA,QAAuB;EACrB,IAAI,CAAC,KAAKD,WAAW;GACnB,MAAM,WAAY,KAAKA,YAAY,SAAe;GAClD,KAAKJ,IAAI,OACP,uBAAuB,KAAKC,aAAa,KAAK,4BAEhD;GACA,KAAK,MAAM,OAAO,KAAKA,cACrB,IAAI,kBAAkB;IAAC,KAAK;IAAU,UAAU;GAAI,CAAC;GAEvD,MAAM,WAAW,KAAKC,mBAElB,KAAKF,IAAI,OACP,yBAAyB,KAAKC,aAAa,KAAK,8BAEhD,MAAM,KAAK,KAAKA,eAAc,QAAO,IAAI,EAAE,CAC7C,GACF,wBACF;GACA,SAAc,QAAQ,cAAc,KAAKE,eAAe,QAAQ,CAAC;GACjE,KAAKE,cAAc;EACrB;EACA,OAAO,KAAKD,UAAU;CACxB;;CAGA,OAAO;EACL,KAAKA,WAAW,QAAQ;CAC1B;CAEA,gBAAgB;EACd,IAAI,KAAKA,aAAa,KAAKH,aAAa,SAAS,GAAG;GAClD,KAAKD,IAAI,OAAO,yBAAyB;GACzC,KAAKI,UAAU,QAAQ;EACzB;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts

@@ -33,9 +33,9 @@ export declare class Forwarder {
      * two components have an equivalent interpretation of whether a Transaction is
      * currently being streamed.
      *
-     * This version of forward is fire-and-forget, with no flow control. The
-     * change-streamer should call and await {@link forwardWithFlowControl()}
-     * occasionally to avoid memory blowup.
+     * This version of forward buffers changes across the current event loop turn
+     * and dispatches them in batches. The change-streamer should call and await
+     * {@link forwardWithFlowControl()} occasionally to avoid memory blowup.
      */
     forward(entry: WatermarkedChange): void;
     sendStatus(status: Status): void;

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAY,MAAM,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE/E,OAAO,KAAK,EAAC,UAAU,EAAkB,MAAM,iBAAiB,CAAC;AAEjE,MAAM,MAAM,sBAAsB,GAAG;IACnC,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5D,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IAC7C,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF,qBAAa,SAAS;;IA+BpB,YACE,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAEL,EA6DF;IAED,oBAAoB,SAMnB;IAED;;;;;;;OAOG;IACH,uBAAuB,CAAC,GAAG,EAAE,MAAM,QA6DlC;IAED,mBAAmB,SAElB;IAED;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU,QAMlB;IAED,MAAM,CAAC,GAAG,EAAE,UAAU,QAIrB;IAED;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB,QAG/B;IAED,UAAU,CAAC,MAAM,EAAE,MAAM,QAIxB;IAED;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB,iBAyBpD;IA6CD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,CAOrB;CACF"}
+{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAY,MAAM,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE/E,OAAO,KAAK,EAAC,UAAU,EAAkB,MAAM,iBAAiB,CAAC;AAEjE,MAAM,MAAM,sBAAsB,GAAG;IACnC,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5D,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IAC7C,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAIF,qBAAa,SAAS;;IAmCpB,YACE,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAEL,EA6DF;IAED,oBAAoB,SAMnB;IAED;;;;;;;OAOG;IACH,uBAAuB,CAAC,GAAG,EAAE,MAAM,QA6DlC;IAED,mBAAmB,SAGlB;IAED;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU,QAOlB;IAED,MAAM,CAAC,GAAG,EAAE,UAAU,QAKrB;IAED;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB,QAmB/B;IAED,UAAU,CAAC,MAAM,EAAE,MAAM,QAKxB;IAED;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB,iBA4BpD;IAkED,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,CAOrB;CACF"}

package/out/zero-cache/src/services/change-streamer/forwarder.js

@@ -2,6 +2,7 @@ import { joinIterables, wrapIterable } from "../../../../shared/src/iterables.js
 import { getOrCreateCounter, getOrCreateGauge, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
 import { Broadcast } from "./broadcast.js";
 //#region ../zero-cache/src/services/change-streamer/forwarder.ts
+var FORWARD_BATCH_SIZE = 64;
 var Forwarder = class {
 	#lc;
 	#setTimeout;
@@ -13,6 +14,9 @@ var Forwarder = class {
 	#flowControlWaits = getOrCreateCounter("replication", "flow_control.waits", "Flow-control checkpoints completed, labeled by release mode.");
 	#flowControlWaitTime = getOrCreateLatencyHistogram("replication", "flow_control.wait_duration", "Time replication waits at flow-control checkpoints.");
 	#inTransaction = false;
+	#pending = [];
+	#flushScheduled = false;
+	#flushImmediateId = void 0;
 	#currentBroadcast;
 	#progressMonitor;
 	#backupLagEventCount = 0;
@@ -79,6 +83,7 @@ var Forwarder = class {
 	}
 	stopProgressMonitor() {
 		clearInterval(this.#progressMonitor);
+		this.#flushPendingWithoutTracking();
 	}
 	/**
 	* `add()` is called in lock step with `Storer.catchup()` so that the
@@ -86,10 +91,12 @@ var Forwarder = class {
 	* currently being streamed.
 	*/
 	add(sub) {
+		this.#flushPendingWithoutTracking();
 		if (this.#inTransaction) this.#queued.add(sub);
 		else this.#active.add(sub);
 	}
 	remove(sub) {
+		this.#flushPendingWithoutTracking();
 		this.#active.delete(sub);
 		this.#queued.delete(sub);
 		sub.close();
@@ -99,15 +106,22 @@ var Forwarder = class {
 	* two components have an equivalent interpretation of whether a Transaction is
 	* currently being streamed.
 	*
-	* This version of forward is fire-and-forget, with no flow control. The
-	* change-streamer should call and await {@link forwardWithFlowControl()}
-	* occasionally to avoid memory blowup.
+	* This version of forward buffers changes across the current event loop turn
+	* and dispatches them in batches. The change-streamer should call and await
+	* {@link forwardWithFlowControl()} occasionally to avoid memory blowup.
 	*/
 	forward(entry) {
-		Broadcast.withoutTracking(this.#active.values(), entry);
+		this.#pending.push(entry);
+		if (this.#queued.size > 0 && (entry[1] === "commit" || entry[1] === "rollback")) this.#flushPendingWithoutTracking();
 		this.#updateActiveSubscribers(entry[1]);
+		if (this.#pending.length >= FORWARD_BATCH_SIZE) this.#flushPendingWithoutTracking();
+		else if (!this.#flushScheduled && this.#pending.length > 0) {
+			this.#flushScheduled = true;
+			this.#flushImmediateId = setImmediate(() => this.#flushPendingWithoutTracking());
+		}
 	}
 	sendStatus(status) {
+		this.#flushPendingWithoutTracking();
 		for (const sub of this.#active.values()) sub.sendStatus(status);
 	}
 	/**
@@ -115,8 +129,10 @@ var Forwarder = class {
 	* Promise that resolves when replication should continue.
 	*/
 	async forwardWithFlowControl(entry) {
+		this.#pending.push(entry);
+		const batch = this.#drainPending();
 		const start = performance.now();
-		const broadcast = new Broadcast(this.#lc, this.#active.values(), entry, this.#earlyReleaseOptions);
+		const broadcast = new Broadcast(this.#lc, this.#active.values(), batch, this.#earlyReleaseOptions);
 		this.#updateActiveSubscribers(entry[1]);
 		this.#currentBroadcast = broadcast;
 		try {
@@ -128,6 +144,21 @@ var Forwarder = class {
 			if (this.#currentBroadcast === broadcast) this.#currentBroadcast = void 0;
 		}
 	}
+	#drainPending() {
+		if (this.#flushImmediateId !== void 0) {
+			clearImmediate(this.#flushImmediateId);
+			this.#flushImmediateId = void 0;
+		}
+		this.#flushScheduled = false;
+		if (this.#pending.length === 0) return [];
+		const batch = this.#pending;
+		this.#pending = [];
+		return batch;
+	}
+	#flushPendingWithoutTracking() {
+		const batch = this.#drainPending();
+		if (batch.length > 0) Broadcast.withoutTracking(this.#active.values(), batch);
+	}
 	#subscriberStats() {
 		let pendingMessages = 0;
 		let backlogMessages = 0;