@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.40

package/out/zero-cache/src/services/change-streamer/subscriber.d.ts

@@ -1,18 +1,52 @@
 import type { Enum } from '../../../../shared/src/enum.ts';
 import type { Subscription } from '../../types/subscription.ts';
 import type { ReplicatorMode } from '../replicator/replicator.ts';
+import type { BackfillState } from './backfill-state.ts';
+import type { PreSerializedBatch } from './broadcast.ts';
 import type { ChangeTag, Status, WatermarkedChange } from './change-streamer.ts';
 import type * as ErrorType from './error-type-enum.ts';
 type ErrorType = Enum<typeof ErrorType>;
 export type SubscriberOptions = {
     backlogHighWaterBytes?: number | undefined;
     backlogLowWaterRatio?: number | undefined;
+    wsBatched?: boolean | undefined;
     /**
      * Called whenever the subscriber's acked watermark advances, i.e. when it
      * has confirmed that a commit was durably applied to its replica. The ACK
      * therefore lags the subscriber's replica rather than leading it.
      */
     onAck?: ((watermark: string) => void) | undefined;
+    /**
+     * The subscriber's pending backfills (protocol v8+), which are tracked
+     * over the changes sent to the subscriber.
+     */
+    backfills?: BackfillState | undefined;
+    /**
+     * Called once the subscriber has been sent all changes up to the head of
+     * the stream, i.e. its catchup and backlog have been flushed. From this
+     * point on, the subscriber's tracked {@link backfills} reflect the same
+     * position in the stream as that of the change-streamer.
+     */
+    onAligned?: ((subscriber: Subscriber) => void) | undefined;
+    /**
+     * Called when a `backfill` or `backfill-completed` message is ignored for
+     * some of the subscriber's pending backfill columns, after the subscriber
+     * is aligned.
+     */
+    onBackfillIgnored?: ((columns: string[]) => void) | undefined;
+    /**
+     * Resolves to whether lag reporting is disabled upstream. If so, a
+     * {@link LAG_REPORTING_DISABLED} status is sent once the subscriber is
+     * aligned, as a stand-in for the lag reports that a subscriber would
+     * otherwise use to determine that it is caught up (e.g. to advertise
+     * readiness to serve traffic).
+     */
+    lagReportingDisabled?: Promise<boolean> | undefined;
+    /**
+     * Whether the subscriber switches over to another replication-manager
+     * when this one drains (see `DrainTracker`).
+     */
+    drainable?: boolean | undefined;
 };
 export type BacklogFullWait = {
     readonly promise: Promise<void>;
@@ -37,7 +71,24 @@ export declare class Subscriber {
     #private;
     readonly id: string;
     readonly mode: ReplicatorMode;
-    constructor(protocolVersion: number, id: string, mode: ReplicatorMode, watermark: string, downstream: Subscription<string>, latestStatus: () => Status, options?: SubscriberOptions);
+    readonly drainable: boolean;
+    constructor(protocolVersion: number, id: string, mode: ReplicatorMode, watermark: string, downstream: Subscription<string | PreSerializedBatch>, options?: SubscriberOptions);
+    /**
+     * The subscriber's tracked pending backfills, or `undefined` if the
+     * subscriber does not report them (i.e. protocol < v8).
+     */
+    get backfills(): BackfillState | undefined;
+    /**
+     * Whether the subscriber has been sent all changes up to the head of the
+     * stream (see {@link SubscriberOptions.onAligned}).
+     */
+    get aligned(): boolean;
+    /**
+     * Resolves once the subscriber has caught up, i.e. once it has acked
+     * (durably applied) all changes up to the head of the stream at the time
+     * it was {@link aligned}. Never resolves if the subscriber is closed first.
+     */
+    get caughtUp(): Promise<void>;
     get watermark(): string;
     get acked(): string;
     /**
@@ -60,7 +111,25 @@ export declare class Subscriber {
      */
     whenBacklogFull(): BacklogFullWait;
     send(change: WatermarkedChange): Promise<void>;
+    sendBatch(changes: readonly WatermarkedChange[], preSerialized?: PreSerializedBatch | undefined): Promise<void>;
+    /**
+     * Sends a status message (e.g. a lag report) downstream. While the
+     * subscriber is catching up, the status is queued in the backlog behind
+     * the changes that preceded it in the replication stream, so that the
+     * subscriber processes it in stream order. This is necessary for lag
+     * reports to measure the lag of the subscriber, including its catchup.
+     *
+     * Status messages received before the subscriber is initialized are
+     * dropped.
+     */
     sendStatus(status: Status): void;
+    /**
+     * Sends a control status (e.g. `draining` or `restore`) downstream
+     * immediately, regardless of whether the subscriber is initialized or
+     * catching up. Unlike {@link sendStatus}, such statuses are not tied to a
+     * position in the replication stream.
+     */
+    sendControlStatus(status: Status): void;
     /** catchup() is called on ChangeEntries loaded from the store. */
     catchup(change: WatermarkedChange): Promise<void>;
     /**
@@ -110,7 +179,16 @@ export declare class Subscriber {
      * Callers are responsible for logging `err`; it is not carried downstream.
      */
     fail(_err?: unknown): void;
-    close(error?: ErrorType, message?: string): void;
+    close(error?: ErrorType, message?: string, details?: {
+        replicaVersion?: string | undefined;
+    }): void;
 }
+/**
+ * Sent to an aligned subscriber when lag reporting is disabled upstream,
+ * signaling that the subscriber is caught up. This is a lag report with a
+ * `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured
+ * (see `isLagReportingDisabledSignal()` in the replicator's recorder).
+ */
+export declare const LAG_REPORTING_DISABLED: Status;
 export {};
 //# sourceMappingURL=subscriber.d.ts.map

package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAKzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EACV,SAAS,EAET,MAAM,EACN,iBAAiB,EAClB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAEvD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAKxC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE1C;;;;OAIG;IACH,KAAK,CAAC,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;CACnD,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,IAAI,IAAI,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEF;;;;;;GAMG;AACH,qBAAa,UAAU;;IAErB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAgB9B,YACE,eAAe,EAAE,MAAM,EACvB,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,cAAc,EACpB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,YAAY,CAAC,MAAM,CAAC,EAChC,YAAY,EAAE,MAAM,MAAM,EAC1B,OAAO,GAAE,iBAAsB,EAehC;IAED,IAAI,SAAS,WAEZ;IAED,IAAI,KAAK,WAER;IAED;;;;;OAKG;IACH,IAAI,WAAW,YAId;IAED;;;OAGG;IACH,YAAY,YAEX;IAED;;;;;OAKG;IACH,eAAe,IAAI,eAAe,CAYjC;IAED,IAAI,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAa7C;IAeD,UAAU,CAAC,MAAM,EAAE,MAAM,QAIxB;IAED,kEAAkE;IAC5D,OAAO,CAAC,MAAM,EAAE,iBAAiB,iBAGtC;IAED;;;OAGG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAc3B;IA2DD;;OAEG;IACH,IAAI,UAAU,WAEb;IAED;;;OAGG;IACH,IAAI,YAAY,WAEf;IAED;;;OAGG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,SAAK,GAAG,IAAI,CAMpD;IAED,QAAQ,IAAI,eAAe,CAyB1B;IAKD,mBAAmB,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW,QAKlD;IAED;;;;;;;;;;OAUG;IACH,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,aAAa,UAW9D;IAED,eAAe,CAAC,GAAG,EAAE,SAAS,WAO7B;IAED;;;;;;;;;;;OAWG;IACH,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,QAElB;IAED,KAAK,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM,QAkBxC;CA0FF"}
+{"version":3,"file":"subscriber.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAKzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EACV,SAAS,EAET,MAAM,EACN,iBAAiB,EAClB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAEvD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAKxC,MAAM,MAAM,iBAAiB,GAAG;IAC9B,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAEhC;;;;OAIG;IACH,KAAK,CAAC,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IAElD;;;OAGG;IACH,SAAS,CAAC,EAAE,aAAa,GAAG,SAAS,CAAC;IAEtC;;;;;OAKG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,UAAU,EAAE,UAAU,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IAE3D;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IAE9D;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC;IAEpD;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACjC,CAAC;AAkBF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,IAAI,IAAI,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEF;;;;;;GAMG;AACH,qBAAa,UAAU;;IAErB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAyB5B,YACE,eAAe,EAAE,MAAM,EACvB,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,cAAc,EACpB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,YAAY,CAAC,MAAM,GAAG,kBAAkB,CAAC,EACrD,OAAO,GAAE,iBAAsB,EAoBhC;IAED;;;OAGG;IACH,IAAI,SAAS,IAAI,aAAa,GAAG,SAAS,CAEzC;IAED;;;OAGG;IACH,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;;;OAIG;IACH,IAAI,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAE5B;IAWD,IAAI,SAAS,WAEZ;IAED,IAAI,KAAK,WAER;IAED;;;;;OAKG;IACH,IAAI,WAAW,YAId;IAED;;;OAGG;IACH,YAAY,YAEX;IAED;;;;;OAKG;IACH,eAAe,IAAI,eAAe,CAYjC;IAED,IAAI,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAa7C;IAED,SAAS,CACP,OAAO,EAAE,SAAS,iBAAiB,EAAE,EACrC,aAAa,CAAC,EAAE,kBAAkB,GAAG,SAAS,GAC7C,OAAO,CAAC,IAAI,CAAC,CA4Cf;IAgBD;;;;;;;;;OASG;IACH,UAAU,CAAC,MAAM,EAAE,MAAM,QAUxB;IAED;;;;;OAKG;IACH,iBAAiB,CAAC,MAAM,EAAE,MAAM,QAE/B;IAED,kEAAkE;IAC5D,OAAO,CAAC,MAAM,EAAE,iBAAiB,iBAGtC;IAED;;;OAGG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAc3B;IA+FD;;OAEG;IACH,IAAI,UAAU,WAEb;IAED;;;OAGG;IACH,IAAI,YAAY,WAEf;IAED;;;OAGG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,SAAK,GAAG,IAAI,CAMpD;IAED,QAAQ,IAAI,eAAe,CAyB1B;IAKD,mBAAmB,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW,QAKlD;IAED;;;;;;;;;;OAUG;IACH,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,aAAa,UAW9D;IAED,eAAe,CAAC,GAAG,EAAE,SAAS,WAO7B;IAED;;;;;;;;;;;OAWG;IACH,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,QAElB;IAED,KAAK,CACH,KAAK,CAAC,EAAE,SAAS,EACjB,OAAO,CAAC,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;QAAC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAC,QAoBhD;CA2GF;AA4CD;;;;;GAKG;AACH,eAAO,MAAM,sBAAsB,EAAE,MAGpC,CAAC"}

package/out/zero-cache/src/services/change-streamer/subscriber.js

@@ -8,6 +8,12 @@ import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-streamer/subscriber.ts
 var DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;
 var DEFAULT_BACKLOG_LOW_WATER_RATIO = .8;
+function isQueuedStatus(entry) {
+	return !Array.isArray(entry);
+}
+function entryBytes(entry) {
+	return isQueuedStatus(entry) ? entry.status.length : entry[2].length;
+}
 /**
 * Encapsulates a subscriber to changes. All subscribers start in a
 * "catchup" phase in which changes are buffered in a backlog while the
@@ -19,8 +25,9 @@ var Subscriber = class {
 	#protocolVersion;
 	id;
 	mode;
+	drainable;
 	#downstream;
-	#latestStatus;
+	#wsBatched;
 	#watermark;
 	#acked;
 	#backlog;
@@ -30,17 +37,57 @@ var Subscriber = class {
 	#backlogBackpressure;
 	#backlogFullWaiters = /* @__PURE__ */ new Set();
 	#onAck;
-	constructor(protocolVersion, id, mode, watermark, downstream, latestStatus, options = {}) {
+	#backfills;
+	#onAligned;
+	#onBackfillIgnored;
+	#lagReportingDisabled;
+	#aligned = false;
+	#caughtUpWatermark;
+	#caughtUp = resolver();
+	constructor(protocolVersion, id, mode, watermark, downstream, options = {}) {
 		this.#protocolVersion = protocolVersion;
 		this.id = id;
 		this.mode = mode;
+		this.drainable = options.drainable ?? false;
 		this.#downstream = downstream;
-		this.#latestStatus = latestStatus;
+		this.#wsBatched = options.wsBatched ?? false;
 		this.#watermark = watermark;
 		this.#acked = watermark;
 		this.#backlog = new RingBuffer();
 		this.#backlogBackpressure = new ByteBackpressureGate(options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES, options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO);
 		this.#onAck = options.onAck;
+		this.#backfills = options.backfills;
+		this.#onAligned = options.onAligned;
+		this.#onBackfillIgnored = options.onBackfillIgnored;
+		this.#lagReportingDisabled = options.lagReportingDisabled;
+	}
+	/**
+	* The subscriber's tracked pending backfills, or `undefined` if the
+	* subscriber does not report them (i.e. protocol < v8).
+	*/
+	get backfills() {
+		return this.#backfills;
+	}
+	/**
+	* Whether the subscriber has been sent all changes up to the head of the
+	* stream (see {@link SubscriberOptions.onAligned}).
+	*/
+	get aligned() {
+		return this.#aligned;
+	}
+	/**
+	* Resolves once the subscriber has caught up, i.e. once it has acked
+	* (durably applied) all changes up to the head of the stream at the time
+	* it was {@link aligned}. Never resolves if the subscriber is closed first.
+	*/
+	get caughtUp() {
+		return this.#caughtUp.promise;
+	}
+	#trackBackfills(change) {
+		if (this.#backfills) {
+			const ignored = this.#backfills.applySerialized(change);
+			if (ignored.length && this.#aligned) this.#onBackfillIgnored?.(ignored);
+		}
 	}
 	get watermark() {
 		return this.#watermark;
@@ -95,6 +142,27 @@ var Subscriber = class {
 		}
 		return promiseVoid;
 	}
+	sendBatch(changes, preSerialized) {
+		if (changes.length === 0) return promiseVoid;
+		if (this.#wsBatched && preSerialized && !this.#backlog && this.#initialized && changes[0][0] > this.#watermark && (this.#protocolVersion >= 5 || changes.every((c) => this.supportsMessage(c[1])))) {
+			let commitWatermark;
+			for (let i = changes.length - 1; i >= 0; i--) if (changes[i][1] === "commit") {
+				commitWatermark = changes[i][0];
+				break;
+			}
+			if (commitWatermark) this.#watermark = commitWatermark;
+			for (const change of changes) this.#trackBackfills(change);
+			return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);
+		}
+		const promises = [];
+		for (const change of changes) {
+			const p = this.send(change);
+			if (p !== promiseVoid) promises.push(p);
+		}
+		if (promises.length === 0) return promiseVoid;
+		if (promises.length === 1) return promises[0];
+		return Promise.all(promises).then(() => {});
+	}
 	#initialized = false;
 	/**
 	* Called once the subscriber's watermark has been validated in the initial
@@ -103,11 +171,36 @@ var Subscriber = class {
 	#initialize() {
 		if (!this.#initialized) {
 			this.#initialized = true;
-			this.sendStatus(this.#latestStatus());
+			this.#sendDownstream(["status", { tag: "status" }]);
 		}
 	}
+	/**
+	* Sends a status message (e.g. a lag report) downstream. While the
+	* subscriber is catching up, the status is queued in the backlog behind
+	* the changes that preceded it in the replication stream, so that the
+	* subscriber processes it in stream order. This is necessary for lag
+	* reports to measure the lag of the subscriber, including its catchup.
+	*
+	* Status messages received before the subscriber is initialized are
+	* dropped.
+	*/
 	sendStatus(status) {
-		if (this.#initialized) this.#sendDownstream(["status", status]);
+		if (!this.#initialized) return;
+		const downstream = ["status", status];
+		if (this.#backlog) {
+			this.#pushBacklog({ status: BigIntJSON.stringify(downstream) });
+			return;
+		}
+		this.#sendDownstream(downstream);
+	}
+	/**
+	* Sends a control status (e.g. `draining` or `restore`) downstream
+	* immediately, regardless of whether the subscriber is initialized or
+	* catching up. Unlike {@link sendStatus}, such statuses are not tied to a
+	* position in the replication stream.
+	*/
+	sendControlStatus(status) {
+		this.#sendDownstream(["status", status]);
 	}
 	/** catchup() is called on ChangeEntries loaded from the store. */
 	async catchup(change) {
@@ -132,13 +225,22 @@ var Subscriber = class {
 		if (watermark <= this.watermark) return;
 		if (!this.supportsMessage(tag)) return;
 		if (tag === "commit") this.#watermark = watermark;
+		this.#trackBackfills(change);
 		const result = await this.#sendStringifiedDownstream(json);
-		if (tag === "commit" && result === "consumed") {
-			const acked = max(this.#acked, watermark);
-			if (acked !== this.#acked) {
-				this.#acked = acked;
-				this.#onAck?.(acked);
-			}
+		if (tag === "commit" && result === "consumed") this.#advanceAck(watermark);
+	}
+	#advanceAck(watermark) {
+		const acked = max(this.#acked, watermark);
+		if (acked !== this.#acked) {
+			this.#acked = acked;
+			this.#onAck?.(acked);
+			this.#checkCaughtUp();
+		}
+	}
+	#checkCaughtUp() {
+		if (this.#caughtUpWatermark !== void 0 && this.#acked >= this.#caughtUpWatermark) {
+			this.#caughtUpWatermark = void 0;
+			this.#caughtUp.resolve();
 		}
 	}
 	#sendDownstream(downstream) {
@@ -157,6 +259,20 @@ var Subscriber = class {
 			this.#processed++;
 		}
 	}
+	async #sendPreSerializedDownstream(batch, commitWatermark) {
+		const size = batch.byteLength;
+		this.#pending += batch.changes.length;
+		this.#pendingBytes += size;
+		const { result } = this.#downstream.push(batch);
+		try {
+			const outcome = await result;
+			if (commitWatermark && outcome === "consumed") this.#advanceAck(commitWatermark);
+		} finally {
+			this.#pending -= batch.changes.length;
+			this.#pendingBytes -= size;
+			this.#processed += batch.changes.length;
+		}
+	}
 	#pending = 0;
 	#pendingBytes = 0;
 	#processed = 0;
@@ -259,14 +375,15 @@ var Subscriber = class {
 	fail(_err) {
 		this.close();
 	}
-	close(error, message) {
+	close(error, message, details) {
 		this.#backlog = null;
 		this.#backlogBytes = 0;
 		this.#backlogBackpressure.releaseAll();
 		this.#resolveBacklogFullWaiters();
 		if (error !== void 0) this.#sendDownstream(["error", {
 			type: error,
-			message
+			message,
+			...details
 		}]).finally(() => this.#downstream.cancel());
 		else this.#downstream.cancel();
 	}
@@ -279,10 +396,10 @@ var Subscriber = class {
 	get #totalBufferedBytes() {
 		return this.#bufferedBacklogBytes + this.#pendingBytes;
 	}
-	#pushBacklog(change) {
+	#pushBacklog(entry) {
 		assert(this.#backlog, "cannot push to backlog after catchup completed");
-		this.#backlog.push(change);
-		this.#backlogBytes += change[2].length;
+		this.#backlog.push(entry);
+		this.#backlogBytes += entryBytes(entry);
 		if (this.backlogFull) this.#resolveBacklogFullWaiters();
 	}
 	#resolveBacklogFullWaiters() {
@@ -298,18 +415,28 @@ var Subscriber = class {
 		let inFlightBytes = 0;
 		try {
 			for (;;) {
-				const change = this.#backlog?.shift();
-				if (!change) {
+				const entry = this.#backlog?.shift();
+				if (!entry) {
+					const closed = this.#backlog === null;
 					this.#backlog = null;
 					this.#backlogBytes = 0;
 					this.#backlogBackpressure.releaseIfUnderLowWater(this.#bufferedBacklogBytes);
+					if (!closed && !this.#aligned) {
+						this.#aligned = true;
+						this.#onAligned?.(this);
+						this.#caughtUpWatermark = this.#watermark;
+						this.#checkCaughtUp();
+						this.#lagReportingDisabled?.then((disabled) => {
+							if (disabled) this.sendStatus(LAG_REPORTING_DISABLED);
+						});
+					}
 					break;
 				}
-				const bytes = change[2].length;
+				const bytes = entryBytes(entry);
 				this.#backlogBytes -= bytes;
 				this.#backlogInFlightBytes += bytes;
 				this.#backlogBackpressure.releaseIfUnderLowWater(this.#bufferedBacklogBytes);
-				const promise = this.#sendChange(change).finally(() => {
+				const promise = (isQueuedStatus(entry) ? this.#sendStringifiedDownstream(entry.status).then(() => {}) : this.#sendChange(entry)).finally(() => {
 					this.#backlogInFlightBytes -= bytes;
 					this.#backlogBackpressure.releaseIfUnderLowWater(this.#bufferedBacklogBytes);
 				});
@@ -354,6 +481,16 @@ var ByteBackpressureGate = class {
 		for (const waiter of waiters) waiter.resolve();
 	}
 };
+/**
+* Sent to an aligned subscriber when lag reporting is disabled upstream,
+* signaling that the subscriber is caught up. This is a lag report with a
+* `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured
+* (see `isLagReportingDisabledSignal()` in the replicator's recorder).
+*/
+var LAG_REPORTING_DISABLED = {
+	tag: "status",
+	lagReport: { nextSendTimeMs: 0 }
+};
 //#endregion
 export { Subscriber };
 

package/out/zero-cache/src/services/change-streamer/subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#latestStatus","#backlogBackpressure","#backlogFullWaiters","#onAck","#watermark","#acked","#backlog","#bufferedBacklogBytes","#pushBacklog","#maybeWaitForBacklogSpace","#sendChange","#initialized","#sendDownstream","#initialize","#backlogDrain","#drainBacklog","#sendStringifiedDownstream","#pending","#pendingBytes","#processed","#backlogCount","#samples","#totalBufferedBytes","#missedLastTimeout","#laggingSinceMs","#backlogBytes","#resolveBacklogFullWaiters","#backlogInFlightBytes","#lowWaterBytes","#waiters"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n};\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly #downstream: Subscription<string>;\n  readonly #latestStatus: () => Status;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<WatermarkedChange> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string>,\n    latestStatus: () => Status,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.#downstream = downstream;\n    this.#latestStatus = latestStatus;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      this.sendStatus(this.#latestStatus());\n    }\n  }\n\n  sendStatus(status: Status) {\n    if (this.#initialized) {\n      void this.#sendDownstream(['status', status]);\n    }\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      // Sends can complete out of order (e.g. the bounded window in\n      // #drainBacklog), so the ack only advances monotonically, and listeners\n      // are only notified when it does.\n      const acked = max(this.#acked, watermark);\n      if (acked !== this.#acked) {\n        this.#acked = acked;\n        this.#onAck?.(acked);\n      }\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(error?: ErrorType, message?: string) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream(['error', {type: error, message}]).finally(() =>\n        this.#downstream.cancel(),\n      );\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(change: WatermarkedChange) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(change);\n    this.#backlogBytes += change[2].length;\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const change = this.#backlog?.shift();\n        if (!change) {\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          break;\n        }\n\n        const bytes = change[2].length;\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const promise = this.#sendChange(change).finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAoBA,IAAM,mCAAmC,KAAK,OAAO;AACrD,IAAM,kCAAkC;;;;;;;;AAmCxC,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CAEA,YACE,iBACA,IACA,MACA,WACA,YACA,cACA,UAA6B,CAAC,GAC9B;EACA,KAAKA,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAKC,cAAc;EACnB,KAAKC,gBAAgB;EACrB,KAAKI,aAAa;EAClB,KAAKC,SAAS;EACd,KAAKC,WAAW,IAAI,WAAW;EAC/B,KAAKL,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAKE,SAAS,QAAQ;CACxB;CAEA,IAAI,YAAY;EACd,OAAO,KAAKC;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAKC;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAKE,yBAAyB,KAAKN,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAKK,aAAa,QAAQ,KAAKC,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAKL,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAKA,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAKE,YAAY;GAC/B,IAAI,KAAKE,UAAU;IAIjB,KAAKE,aAAa,MAAM;IACxB,OAAO,KAAKC,0BAA0B;GACxC;GACA,OAAO,KAAKC,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAKC,cAAc;GACtB,KAAKA,eAAe;GACpB,KAAK,WAAW,KAAKX,cAAc,CAAC;EACtC;CACF;CAEA,WAAW,QAAgB;EACzB,IAAI,KAAKW,cACP,KAAUC,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAEhD;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAKC,YAAY;EACjB,MAAM,KAAKH,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAKG,YAAY;EACjB,IAAI,CAAC,KAAKP,UACR,OAAO,KAAKQ,iBAAiB;EAE/B,IAAI,CAAC,KAAKA,eAAe;GAKvB,KAAKA,gBAAgB,KAAKC,cAAc;GACxC,KAAUD,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAKA;CACd;CAEA,MAAMJ,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAKN,aAAa;EAEpB,MAAM,SAAS,MAAM,KAAKY,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YAAY;GAI7C,MAAM,QAAQ,IAAI,KAAKX,QAAQ,SAAS;GACxC,IAAI,UAAU,KAAKA,QAAQ;IACzB,KAAKA,SAAS;IACd,KAAKF,SAAS,KAAK;GACrB;EACF;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAKa,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAMA,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAKC;EACL,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAKnB,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAKkB;GACL,KAAKC,iBAAiB;GACtB,KAAKC;EACP;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAKF,WAAW,KAAKG;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAKD;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAKE,SAAS,UAAU,YAC7B,KAAKA,SAAS,MAAM;EAEtB,KAAKA,SAAS,KAAK;GAAC,WAAW,KAAKF;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAKE,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAKD;GACd,cAAc,KAAKb;GACnB,oBAAoB,KAAKe;GACzB,mBAAmB,KAAKC;EAC1B;EAEF,MAAM,OAAO,KAAKF,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAKA,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAKD;GACd,cAAc,KAAKb;GACnB,oBAAoB,KAAKe;GACzB,mBAAmB,KAAKC;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAKA,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAKC,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAKD,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAKC,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAKA,oBAAoB;EACzB,OAAO,MAAM,KAAKA;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAK1B,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MAAM,OAAmB,SAAkB;EAGzC,KAAKQ,WAAW;EAChB,KAAKmB,gBAAgB;EACrB,KAAKxB,qBAAqB,WAAW;EAGrC,KAAKyB,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAUd,gBAAgB,CAAC,SAAS;GAAC,MAAM;GAAO;EAAO,CAAC,CAAC,EAAE,cAC3D,KAAKb,YAAY,OAAO,CAC1B;OAEA,KAAKA,YAAY,OAAO;CAE5B;CAEA,IAAIqB,gBAAgB;EAClB,OAAO,KAAKd,UAAU,QAAQ;CAChC;CAEA,IAAIC,wBAAwB;EAI1B,OAAO,KAAKkB,gBAAgB,KAAKE;CACnC;CAEA,IAAIL,sBAAsB;EACxB,OAAO,KAAKf,wBAAwB,KAAKW;CAC3C;CAEA,aAAa,QAA2B;EACtC,OAAO,KAAKZ,UAAU,gDAAgD;EACtE,KAAKA,SAAS,KAAK,MAAM;EACzB,KAAKmB,iBAAiB,OAAO,GAAG;EAChC,IAAI,KAAK,aACP,KAAKC,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAKxB,mBAAmB;EAC5C,KAAKA,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAKD,qBAAqB,aAAa,KAAKM,qBAAqB;CAC1E;CAEA,MAAMQ,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,SAAS,KAAKT,UAAU,MAAM;IACpC,IAAI,CAAC,QAAQ;KACX,KAAKA,WAAW;KAChB,KAAKmB,gBAAgB;KACrB,KAAKxB,qBAAqB,uBACxB,KAAKM,qBACP;KACA;IACF;IAEA,MAAM,QAAQ,OAAO,GAAG;IACxB,KAAKkB,iBAAiB;IACtB,KAAKE,yBAAyB;IAC9B,KAAK1B,qBAAqB,uBACxB,KAAKM,qBACP;IAKA,MAAM,UAAU,KAAKG,YAAY,MAAM,EAAE,cAAc;KACrD,KAAKiB,yBAAyB;KAC9B,KAAK1B,qBAAqB,uBACxB,KAAKM,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAKN,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAKa,gBAAgB;GACrB,KAAKb,qBAAqB,uBACxB,KAAKM,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAKqB,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAKC,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAKA,SAAS,WAAW,KAAK,gBAAgB,KAAKD,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAKC,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF"}
+{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#wsBatched","#backlogBackpressure","#backlogFullWaiters","#onAck","#backfills","#onAligned","#onBackfillIgnored","#lagReportingDisabled","#caughtUp","#watermark","#acked","#backlog","#aligned","#bufferedBacklogBytes","#pushBacklog","#maybeWaitForBacklogSpace","#sendChange","#initialized","#trackBackfills","#sendPreSerializedDownstream","#sendDownstream","#initialize","#backlogDrain","#drainBacklog","#sendStringifiedDownstream","#advanceAck","#checkCaughtUp","#caughtUpWatermark","#pending","#pendingBytes","#processed","#backlogCount","#samples","#totalBufferedBytes","#missedLastTimeout","#laggingSinceMs","#backlogBytes","#resolveBacklogFullWaiters","#backlogInFlightBytes","#lowWaterBytes","#waiters"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n  wsBatched?: boolean | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n\n  /**\n   * The subscriber's pending backfills (protocol v8+), which are tracked\n   * over the changes sent to the subscriber.\n   */\n  backfills?: BackfillState | undefined;\n\n  /**\n   * Called once the subscriber has been sent all changes up to the head of\n   * the stream, i.e. its catchup and backlog have been flushed. From this\n   * point on, the subscriber's tracked {@link backfills} reflect the same\n   * position in the stream as that of the change-streamer.\n   */\n  onAligned?: ((subscriber: Subscriber) => void) | undefined;\n\n  /**\n   * Called when a `backfill` or `backfill-completed` message is ignored for\n   * some of the subscriber's pending backfill columns, after the subscriber\n   * is aligned.\n   */\n  onBackfillIgnored?: ((columns: string[]) => void) | undefined;\n\n  /**\n   * Resolves to whether lag reporting is disabled upstream. If so, a\n   * {@link LAG_REPORTING_DISABLED} status is sent once the subscriber is\n   * aligned, as a stand-in for the lag reports that a subscriber would\n   * otherwise use to determine that it is caught up (e.g. to advertise\n   * readiness to serve traffic).\n   */\n  lagReportingDisabled?: Promise<boolean> | undefined;\n\n  /**\n   * Whether the subscriber switches over to another replication-manager\n   * when this one drains (see `DrainTracker`).\n   */\n  drainable?: boolean | undefined;\n};\n\n/**\n * A status message queued in the backlog, so that it is sent downstream in\n * the order in which it was received relative to (backlogged) changes.\n */\ntype QueuedStatus = {readonly status: string};\n\ntype BacklogEntry = WatermarkedChange | QueuedStatus;\n\nfunction isQueuedStatus(entry: BacklogEntry): entry is QueuedStatus {\n  return !Array.isArray(entry);\n}\n\nfunction entryBytes(entry: BacklogEntry): number {\n  return isQueuedStatus(entry) ? entry.status.length : entry[2].length;\n}\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly drainable: boolean;\n  readonly #downstream: Subscription<string | PreSerializedBatch>;\n  readonly #wsBatched: boolean;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<BacklogEntry> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n  readonly #backfills: BackfillState | undefined;\n  readonly #onAligned: ((subscriber: Subscriber) => void) | undefined;\n  readonly #onBackfillIgnored: ((columns: string[]) => void) | undefined;\n  readonly #lagReportingDisabled: Promise<boolean> | undefined;\n  #aligned = false;\n  // The watermark to be acked for the subscriber to be caught up, set when\n  // it is aligned.\n  #caughtUpWatermark: string | undefined;\n  readonly #caughtUp = resolver<void>();\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string | PreSerializedBatch>,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.drainable = options.drainable ?? false;\n    this.#downstream = downstream;\n    this.#wsBatched = options.wsBatched ?? false;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n    this.#backfills = options.backfills;\n    this.#onAligned = options.onAligned;\n    this.#onBackfillIgnored = options.onBackfillIgnored;\n    this.#lagReportingDisabled = options.lagReportingDisabled;\n  }\n\n  /**\n   * The subscriber's tracked pending backfills, or `undefined` if the\n   * subscriber does not report them (i.e. protocol < v8).\n   */\n  get backfills(): BackfillState | undefined {\n    return this.#backfills;\n  }\n\n  /**\n   * Whether the subscriber has been sent all changes up to the head of the\n   * stream (see {@link SubscriberOptions.onAligned}).\n   */\n  get aligned(): boolean {\n    return this.#aligned;\n  }\n\n  /**\n   * Resolves once the subscriber has caught up, i.e. once it has acked\n   * (durably applied) all changes up to the head of the stream at the time\n   * it was {@link aligned}. Never resolves if the subscriber is closed first.\n   */\n  get caughtUp(): Promise<void> {\n    return this.#caughtUp.promise;\n  }\n\n  #trackBackfills(change: WatermarkedChange) {\n    if (this.#backfills) {\n      const ignored = this.#backfills.applySerialized(change);\n      if (ignored.length && this.#aligned) {\n        this.#onBackfillIgnored?.(ignored);\n      }\n    }\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  sendBatch(\n    changes: readonly WatermarkedChange[],\n    preSerialized?: PreSerializedBatch | undefined,\n  ): Promise<void> {\n    if (changes.length === 0) {\n      return promiseVoid;\n    }\n\n    if (\n      this.#wsBatched &&\n      preSerialized &&\n      !this.#backlog &&\n      this.#initialized &&\n      changes[0][0] > this.#watermark &&\n      (this.#protocolVersion >= 5 ||\n        changes.every(c => this.supportsMessage(c[1])))\n    ) {\n      let commitWatermark: string | undefined;\n      for (let i = changes.length - 1; i >= 0; i--) {\n        if (changes[i][1] === 'commit') {\n          commitWatermark = changes[i][0];\n          break;\n        }\n      }\n      if (commitWatermark) {\n        this.#watermark = commitWatermark;\n      }\n      for (const change of changes) {\n        this.#trackBackfills(change);\n      }\n      return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n    }\n\n    const promises: Promise<void>[] = [];\n    for (const change of changes) {\n      const p = this.send(change);\n      if (p !== promiseVoid) {\n        promises.push(p);\n      }\n    }\n    if (promises.length === 0) {\n      return promiseVoid;\n    }\n    if (promises.length === 1) {\n      return promises[0];\n    }\n    return Promise.all(promises).then(() => {});\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      // The initial status precedes the catchup, so it bypasses the backlog.\n      void this.#sendDownstream(['status', {tag: 'status'}]);\n    }\n  }\n\n  /**\n   * Sends a status message (e.g. a lag report) downstream. While the\n   * subscriber is catching up, the status is queued in the backlog behind\n   * the changes that preceded it in the replication stream, so that the\n   * subscriber processes it in stream order. This is necessary for lag\n   * reports to measure the lag of the subscriber, including its catchup.\n   *\n   * Status messages received before the subscriber is initialized are\n   * dropped.\n   */\n  sendStatus(status: Status) {\n    if (!this.#initialized) {\n      return;\n    }\n    const downstream: Downstream = ['status', status];\n    if (this.#backlog) {\n      this.#pushBacklog({status: BigIntJSON.stringify(downstream)});\n      return;\n    }\n    void this.#sendDownstream(downstream);\n  }\n\n  /**\n   * Sends a control status (e.g. `draining` or `restore`) downstream\n   * immediately, regardless of whether the subscriber is initialized or\n   * catching up. Unlike {@link sendStatus}, such statuses are not tied to a\n   * position in the replication stream.\n   */\n  sendControlStatus(status: Status) {\n    void this.#sendDownstream(['status', status]);\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    this.#trackBackfills(change);\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      this.#advanceAck(watermark);\n    }\n  }\n\n  #advanceAck(watermark: string) {\n    // Sends can complete out of order (e.g. the bounded window in\n    // #drainBacklog), so the ack only advances monotonically, and listeners\n    // are only notified when it does.\n    const acked = max(this.#acked, watermark);\n    if (acked !== this.#acked) {\n      this.#acked = acked;\n      this.#onAck?.(acked);\n      this.#checkCaughtUp();\n    }\n  }\n\n  #checkCaughtUp() {\n    if (\n      this.#caughtUpWatermark !== undefined &&\n      this.#acked >= this.#caughtUpWatermark\n    ) {\n      this.#caughtUpWatermark = undefined;\n      this.#caughtUp.resolve();\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  async #sendPreSerializedDownstream(\n    batch: PreSerializedBatch,\n    commitWatermark?: string | undefined,\n  ): Promise<void> {\n    const size = batch.byteLength;\n    this.#pending += batch.changes.length;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(batch);\n    try {\n      const outcome = await result;\n      if (commitWatermark && outcome === 'consumed') {\n        this.#advanceAck(commitWatermark);\n      }\n    } finally {\n      this.#pending -= batch.changes.length;\n      this.#pendingBytes -= size;\n      this.#processed += batch.changes.length;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(\n    error?: ErrorType,\n    message?: string,\n    details?: {replicaVersion?: string | undefined},\n  ) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream([\n        'error',\n        {type: error, message, ...details},\n      ]).finally(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(entry: BacklogEntry) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(entry);\n    this.#backlogBytes += entryBytes(entry);\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const entry = this.#backlog?.shift();\n        if (!entry) {\n          const closed = this.#backlog === null;\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          if (!closed && !this.#aligned) {\n            // All changes up to the head have been handed to #sendChange()\n            // (and thus tracked), and subsequent changes are sent directly.\n            this.#aligned = true;\n            this.#onAligned?.(this);\n            this.#caughtUpWatermark = this.#watermark;\n            this.#checkCaughtUp();\n            void this.#lagReportingDisabled?.then(disabled => {\n              if (disabled) {\n                this.sendStatus(LAG_REPORTING_DISABLED);\n              }\n            });\n          }\n          break;\n        }\n\n        const bytes = entryBytes(entry);\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const send = isQueuedStatus(entry)\n          ? this.#sendStringifiedDownstream(entry.status).then(() => {})\n          : this.#sendChange(entry);\n        const promise = send.finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n\n/**\n * Sent to an aligned subscriber when lag reporting is disabled upstream,\n * signaling that the subscriber is caught up. This is a lag report with a\n * `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured\n * (see `isLagReportingDisabledSignal()` in the replicator's recorder).\n */\nexport const LAG_REPORTING_DISABLED: Status = {\n  tag: 'status',\n  lagReport: {nextSendTimeMs: 0},\n};\n"],"mappings":";;;;;;;;AAsBA,IAAM,mCAAmC,KAAK,OAAO;AACrD,IAAM,kCAAkC;AA2DxC,SAAS,eAAe,OAA4C;CAClE,OAAO,CAAC,MAAM,QAAQ,KAAK;AAC7B;AAEA,SAAS,WAAW,OAA6B;CAC/C,OAAO,eAAe,KAAK,IAAI,MAAM,OAAO,SAAS,MAAM,GAAG;AAChE;;;;;;;;AAuBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CACA;CACA;CACA;CACA;CACA,WAAW;CAGX;CACA,YAAqB,SAAe;CAEpC,YACE,iBACA,IACA,MACA,WACA,YACA,UAA6B,CAAC,GAC9B;EACA,KAAKA,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAK,YAAY,QAAQ,aAAa;EACtC,KAAKC,cAAc;EACnB,KAAKC,aAAa,QAAQ,aAAa;EACvC,KAAKS,aAAa;EAClB,KAAKC,SAAS;EACd,KAAKC,WAAW,IAAI,WAAW;EAC/B,KAAKV,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAKE,SAAS,QAAQ;EACtB,KAAKC,aAAa,QAAQ;EAC1B,KAAKC,aAAa,QAAQ;EAC1B,KAAKC,qBAAqB,QAAQ;EAClC,KAAKC,wBAAwB,QAAQ;CACvC;;;;;CAMA,IAAI,YAAuC;EACzC,OAAO,KAAKH;CACd;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAKQ;CACd;;;;;;CAOA,IAAI,WAA0B;EAC5B,OAAO,KAAKJ,UAAU;CACxB;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAKJ,YAAY;GACnB,MAAM,UAAU,KAAKA,WAAW,gBAAgB,MAAM;GACtD,IAAI,QAAQ,UAAU,KAAKQ,UACzB,KAAKN,qBAAqB,OAAO;EAErC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAKG;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAKC;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAKG,yBAAyB,KAAKZ,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAKU,aAAa,QAAQ,KAAKE,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAKX,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAKA,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAKO,YAAY;GAC/B,IAAI,KAAKE,UAAU;IAIjB,KAAKG,aAAa,MAAM;IACxB,OAAO,KAAKC,0BAA0B;GACxC;GACA,OAAO,KAAKC,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAKhB,cACL,iBACA,CAAC,KAAKW,YACN,KAAKM,gBACL,QAAQ,GAAG,KAAK,KAAKR,eACpB,KAAKX,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,GAAG,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,GAAG;IAC7B;GACF;GAEF,IAAI,iBACF,KAAKW,aAAa;GAEpB,KAAK,MAAM,UAAU,SACnB,KAAKS,gBAAgB,MAAM;GAE7B,OAAO,KAAKC,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,EAAE,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAKF,cAAc;GACtB,KAAKA,eAAe;GAEpB,KAAUG,gBAAgB,CAAC,UAAU,EAAC,KAAK,SAAQ,CAAC,CAAC;EACvD;CACF;;;;;;;;;;;CAYA,WAAW,QAAgB;EACzB,IAAI,CAAC,KAAKH,cACR;EAEF,MAAM,aAAyB,CAAC,UAAU,MAAM;EAChD,IAAI,KAAKN,UAAU;GACjB,KAAKG,aAAa,EAAC,QAAQ,WAAW,UAAU,UAAU,EAAC,CAAC;GAC5D;EACF;EACA,KAAUM,gBAAgB,UAAU;CACtC;;;;;;;CAQA,kBAAkB,QAAgB;EAChC,KAAUA,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAC9C;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAKC,YAAY;EACjB,MAAM,KAAKL,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAKK,YAAY;EACjB,IAAI,CAAC,KAAKV,UACR,OAAO,KAAKW,iBAAiB;EAE/B,IAAI,CAAC,KAAKA,eAAe;GAKvB,KAAKA,gBAAgB,KAAKC,cAAc;GACxC,KAAUD,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAKA;CACd;CAEA,MAAMN,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAKP,aAAa;EAEpB,KAAKS,gBAAgB,MAAM;EAC3B,MAAM,SAAS,MAAM,KAAKM,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YACjC,KAAKC,YAAY,SAAS;CAE9B;CAEA,YAAY,WAAmB;EAI7B,MAAM,QAAQ,IAAI,KAAKf,QAAQ,SAAS;EACxC,IAAI,UAAU,KAAKA,QAAQ;GACzB,KAAKA,SAAS;GACd,KAAKP,SAAS,KAAK;GACnB,KAAKuB,eAAe;EACtB;CACF;CAEA,iBAAiB;EACf,IACE,KAAKC,uBAAuB,KAAA,KAC5B,KAAKjB,UAAU,KAAKiB,oBACpB;GACA,KAAKA,qBAAqB,KAAA;GAC1B,KAAKnB,UAAU,QAAQ;EACzB;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAKgB,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAMA,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAKI;EACL,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK9B,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAK6B;GACL,KAAKC,iBAAiB;GACtB,KAAKC;EACP;CACF;CAEA,MAAMX,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAKS,YAAY,MAAM,QAAQ;EAC/B,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK9B,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YACjC,KAAK0B,YAAY,eAAe;EAEpC,UAAU;GACR,KAAKG,YAAY,MAAM,QAAQ;GAC/B,KAAKC,iBAAiB;GACtB,KAAKC,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAKF,WAAW,KAAKG;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAKD;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAKE,SAAS,UAAU,YAC7B,KAAKA,SAAS,MAAM;EAEtB,KAAKA,SAAS,KAAK;GAAC,WAAW,KAAKF;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAKE,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAKD;GACd,cAAc,KAAKlB;GACnB,oBAAoB,KAAKoB;GACzB,mBAAmB,KAAKC;EAC1B;EAEF,MAAM,OAAO,KAAKF,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAKA,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAKD;GACd,cAAc,KAAKlB;GACnB,oBAAoB,KAAKoB;GACzB,mBAAmB,KAAKC;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAKA,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAKC,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAKD,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAKC,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAKA,oBAAoB;EACzB,OAAO,MAAM,KAAKA;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAKrC,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MACE,OACA,SACA,SACA;EAGA,KAAKa,WAAW;EAChB,KAAKyB,gBAAgB;EACrB,KAAKnC,qBAAqB,WAAW;EAGrC,KAAKoC,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAUjB,gBAAgB,CACxB,SACA;GAAC,MAAM;GAAO;GAAS,GAAG;EAAO,CACnC,CAAC,EAAE,cAAc,KAAKrB,YAAY,OAAO,CAAC;OAE1C,KAAKA,YAAY,OAAO;CAE5B;CAEA,IAAIgC,gBAAgB;EAClB,OAAO,KAAKpB,UAAU,QAAQ;CAChC;CAEA,IAAIE,wBAAwB;EAI1B,OAAO,KAAKuB,gBAAgB,KAAKE;CACnC;CAEA,IAAIL,sBAAsB;EACxB,OAAO,KAAKpB,wBAAwB,KAAKgB;CAC3C;CAEA,aAAa,OAAqB;EAChC,OAAO,KAAKlB,UAAU,gDAAgD;EACtE,KAAKA,SAAS,KAAK,KAAK;EACxB,KAAKyB,iBAAiB,WAAW,KAAK;EACtC,IAAI,KAAK,aACP,KAAKC,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAKnC,mBAAmB;EAC5C,KAAKA,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAKD,qBAAqB,aAAa,KAAKY,qBAAqB;CAC1E;CAEA,MAAMU,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,QAAQ,KAAKZ,UAAU,MAAM;IACnC,IAAI,CAAC,OAAO;KACV,MAAM,SAAS,KAAKA,aAAa;KACjC,KAAKA,WAAW;KAChB,KAAKyB,gBAAgB;KACrB,KAAKnC,qBAAqB,uBACxB,KAAKY,qBACP;KACA,IAAI,CAAC,UAAU,CAAC,KAAKD,UAAU;MAG7B,KAAKA,WAAW;MAChB,KAAKP,aAAa,IAAI;MACtB,KAAKsB,qBAAqB,KAAKlB;MAC/B,KAAKiB,eAAe;MACpB,KAAUnB,uBAAuB,MAAK,aAAY;OAChD,IAAI,UACF,KAAK,WAAW,sBAAsB;MAE1C,CAAC;KACH;KACA;IACF;IAEA,MAAM,QAAQ,WAAW,KAAK;IAC9B,KAAK6B,iBAAiB;IACtB,KAAKE,yBAAyB;IAC9B,KAAKrC,qBAAqB,uBACxB,KAAKY,qBACP;IAQA,MAAM,WAHO,eAAe,KAAK,IAC7B,KAAKW,2BAA2B,MAAM,MAAM,EAAE,WAAW,CAAC,CAAC,IAC3D,KAAKR,YAAY,KAAK,GACL,cAAc;KACjC,KAAKsB,yBAAyB;KAC9B,KAAKrC,qBAAqB,uBACxB,KAAKY,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAKZ,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAKqB,gBAAgB;GACrB,KAAKrB,qBAAqB,uBACxB,KAAKY,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAK0B,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAKC,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAKA,SAAS,WAAW,KAAK,gBAAgB,KAAKD,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAKC,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF;;;;;;;AAQA,IAAa,yBAAiC;CAC5C,KAAK;CACL,WAAW,EAAC,gBAAgB,EAAC;AAC/B"}

package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts

@@ -25,7 +25,25 @@ type Opts = {
 export declare class UpstreamAcker {
     #private;
     constructor({ trackPgChangeLog, trackBackup }: Opts);
-    reset(upstream: Sink<ChangeSourceUpstream>): void;
+    /**
+     * Starts tracking a new upstream connection, which resumes the change stream
+     * after `resumeWatermark`.
+     *
+     * Everything at or before `resumeWatermark` was committed on an earlier
+     * connection, so it counts as committed on this one: a status watermark past
+     * it waits for the tracked stores to reach it, as it would for a commit on
+     * this connection. Otherwise the first keepalive of a stream that resumes
+     * ahead of the stores -- a change log ahead of its backup, which is the
+     * normal state -- would move the replication slot past transactions that the
+     * backup has not persisted, and a task restored from the backup would then
+     * resume below the slot, which the upstream moves forward without a word.
+     *
+     * When the PG change log is tracked, the stream resumes from its head, so
+     * `resumeWatermark` has been persisted in it (by this or another
+     * change-streamer). Without this, status messages would not be acked on an
+     * idle upstream until a commit is stored.
+     */
+    reset(upstream: Sink<ChangeSourceUpstream>, resumeWatermark: string): void;
     trackDownstream(downstream: ChangeStreamMessage): void;
     trackPgChangeLog(committedWatermark: string): void;
     trackBackup(backedUpWatermark: string): void;

package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"upstream-acker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/upstream-acker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,wBAAwB,CAAC;AACjD,OAAO,KAAK,EACV,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,sCAAsC,CAAC;AAE9C,KAAK,IAAI,GAAG;IACV,gBAAgB,EAAE,OAAO,CAAC;IAC1B,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,aAAa;;IAYxB,YAAY,EAAC,gBAAgB,EAAE,WAAW,EAAC,EAAE,IAAI,EAOhD;IAED,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,oBAAoB,CAAC,QAKzC;IAED,eAAe,CAAC,UAAU,EAAE,mBAAmB,QAc9C;IAED,gBAAgB,CAAC,kBAAkB,EAAE,MAAM,QAO1C;IAED,WAAW,CAAC,iBAAiB,EAAE,MAAM,QAIpC;CA2BF"}
+{"version":3,"file":"upstream-acker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/upstream-acker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,wBAAwB,CAAC;AACjD,OAAO,KAAK,EACV,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,sCAAsC,CAAC;AAE9C,KAAK,IAAI,GAAG;IACV,gBAAgB,EAAE,OAAO,CAAC;IAC1B,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,aAAa;;IAYxB,YAAY,EAAC,gBAAgB,EAAE,WAAW,EAAC,EAAE,IAAI,EAOhD;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,oBAAoB,CAAC,EAAE,eAAe,EAAE,MAAM,QAalE;IAED,eAAe,CAAC,UAAU,EAAE,mBAAmB,QAe9C;IAED,gBAAgB,CAAC,kBAAkB,EAAE,MAAM,QAO1C;IAED,WAAW,CAAC,iBAAiB,EAAE,MAAM,QAIpC;CA2BF"}

package/out/zero-cache/src/services/change-streamer/upstream-acker.js

@@ -33,11 +33,31 @@ var UpstreamAcker = class {
 		this.#trackPgChangeLog = trackPgChangeLog;
 		this.#trackBackup = trackBackup;
 	}
-	reset(upstream) {
+	/**
+	* Starts tracking a new upstream connection, which resumes the change stream
+	* after `resumeWatermark`.
+	*
+	* Everything at or before `resumeWatermark` was committed on an earlier
+	* connection, so it counts as committed on this one: a status watermark past
+	* it waits for the tracked stores to reach it, as it would for a commit on
+	* this connection. Otherwise the first keepalive of a stream that resumes
+	* ahead of the stores -- a change log ahead of its backup, which is the
+	* normal state -- would move the replication slot past transactions that the
+	* backup has not persisted, and a task restored from the backup would then
+	* resume below the slot, which the upstream moves forward without a word.
+	*
+	* When the PG change log is tracked, the stream resumes from its head, so
+	* `resumeWatermark` has been persisted in it (by this or another
+	* change-streamer). Without this, status messages would not be acked on an
+	* idle upstream until a commit is stored.
+	*/
+	reset(upstream, resumeWatermark) {
 		this.#upstream = upstream;
-		this.#lastTx = "";
+		this.#lastTx = resumeWatermark;
 		this.#lastStatus = "";
 		this.#lastAck = "";
+		if (this.#trackPgChangeLog) this.#pgChangeLogWatermark = max(resumeWatermark, this.#pgChangeLogWatermark);
+		this.#maybeAck();
 	}
 	trackDownstream(downstream) {
 		const [tag, msg] = downstream;
@@ -47,7 +67,7 @@ var UpstreamAcker = class {
 				this.#maybeAck();
 				break;
 			case "commit":
-				this.#lastTx = downstream[2].watermark;
+				this.#lastTx = max(this.#lastTx, downstream[2].watermark);
 				this.#maybeAck();
 				break;
 		}