@rocicorp/zero 1.11.0-canary.2 → 1.11.0-canary.20

package/out/zero-cache/src/services/change-streamer/backup-monitor.js

@@ -11,6 +11,14 @@ var BackupMonitor = class {
 	#replicaFile;
 	#firstBackupReceived = resolver();
 	#latestBackup;
+	#firstBackupResolved = false;
+	/**
+	* The replica's `stateVersion` when the monitor started, i.e. the point the
+	* backup has to reach before it covers what this task is about to serve.
+	* `undefined` when the replica could not be read, which degrades the gate to
+	* "any watermark" rather than hanging startup forever.
+	*/
+	#coverageTarget;
 	constructor(lc, watermarks, changeStreamer, replicaFile) {
 		this.#lc = lc;
 		this.#watermarks = watermarks;
@@ -23,21 +31,64 @@ var BackupMonitor = class {
 	async run() {
 		this.#lc.info?.("starting backup monitor");
 		this.#initBackupLagMetric();
+		this.#coverageTarget = this.#readReplicaStateVersion();
 		for await (const backedUp of this.#watermarks) {
-			if (!this.#latestBackup) this.#firstBackupReceived.resolve();
-			else if (backedUp.watermark < this.#latestBackup.watermark) {
-				this.#lc.warn?.(`ignoring earlier backup watermark`, { backedUp });
-				continue;
-			} else if (backedUp.watermark === this.#latestBackup.watermark) {
-				this.#lc.debug?.(`ignoring redundant backup watermark`, { backedUp });
-				continue;
+			if (this.#latestBackup) {
+				if (backedUp.watermark < this.#latestBackup.watermark) {
+					this.#lc.warn?.(`ignoring earlier backup watermark`, { backedUp });
+					continue;
+				}
+				if (backedUp.watermark === this.#latestBackup.watermark) {
+					this.#lc.debug?.(`ignoring redundant backup watermark`, { backedUp });
+					continue;
+				}
 			}
 			this.#lc.info?.(`received backup watermark`, { backedUp });
 			this.#latestBackup = backedUp;
 			this.#changeStreamer.trackBackupWatermark(backedUp.watermark);
+			this.#checkFirstBackupCovers(backedUp);
 		}
 		this.#lc.info?.("watermark stream closed. BackupMonitor stopped.");
 	}
+	/**
+	* Resolves {@link firstBackupReceived} once the backup actually covers the
+	* replica, rather than on the first watermark of any value.
+	*
+	* A new backup destination starts empty and is filled by re-uploading the
+	* local LTX chain, so the first watermarks read back out of it are real but
+	* far behind the replica. Releasing the readiness gate on one of those lets
+	* the task serve against a backup that does not yet cover it, which is what
+	* demotes a restoring view-syncer to Postgres catchup.
+	*
+	* The target is pinned at startup rather than compared against the replica's
+	* current position, which keeps the gate satisfiable under sustained writes.
+	*/
+	#checkFirstBackupCovers(backedUp) {
+		if (this.#firstBackupResolved) return;
+		const target = this.#coverageTarget;
+		if (target !== void 0 && backedUp.watermark < target) {
+			this.#lc.info?.(`backup does not yet cover the replica`, {
+				backedUp,
+				coverageTarget: target
+			});
+			return;
+		}
+		this.#firstBackupResolved = true;
+		this.#firstBackupReceived.resolve();
+	}
+	#readReplicaStateVersion() {
+		let db;
+		try {
+			db = new Database(this.#lc, this.#replicaFile, { readonly: true });
+			const { stateVersion } = db.prepare(`SELECT stateVersion FROM "_zero.replicationState"`).get();
+			return stateVersion;
+		} catch (e) {
+			this.#lc.warn?.("unable to read the replica's stateVersion; the initial backup gate will accept the first watermark", e);
+			return;
+		} finally {
+			db?.close();
+		}
+	}
 	stop() {
 		this.#watermarks.cancel();
 		return promiseVoid;

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#latestBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debuggin; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n\n    for await (const backedUp of this.#watermarks) {\n      if (!this.#latestBackup) {\n        this.#firstBackupReceived.resolve();\n      } else if (backedUp.watermark < this.#latestBackup.watermark) {\n        this.#lc.warn?.(`ignoring earlier backup watermark`, {backedUp});\n        continue;\n      } else if (backedUp.watermark === this.#latestBackup.watermark) {\n        this.#lc.debug?.(`ignoring redundant backup watermark`, {backedUp});\n        continue;\n      }\n      this.#lc.info?.(`received backup watermark`, {backedUp});\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAKC,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAKJ,IAAI,OAAO,yBAAyB;EACzC,KAAKK,qBAAqB;EAE1B,WAAW,MAAM,YAAY,KAAKJ,aAAa;GAC7C,IAAI,CAAC,KAAKK,eACR,KAAKF,qBAAqB,QAAQ;QAC7B,IAAI,SAAS,YAAY,KAAKE,cAAc,WAAW;IAC5D,KAAKN,IAAI,OAAO,qCAAqC,EAAC,SAAQ,CAAC;IAC/D;GACF,OAAO,IAAI,SAAS,cAAc,KAAKM,cAAc,WAAW;IAC9D,KAAKN,IAAI,QAAQ,uCAAuC,EAAC,SAAQ,CAAC;IAClE;GACF;GACA,KAAKA,IAAI,OAAO,6BAA6B,EAAC,SAAQ,CAAC;GACvD,KAAKM,gBAAgB;GACrB,KAAKJ,gBAAgB,qBAAqB,SAAS,SAAS;EAC9D;EACA,KAAKF,IAAI,OAAO,iDAAiD;CACnE;CAEA,OAAsB;EACpB,KAAKC,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,eAAe,KAAKK;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAKN,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKH,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}
+{"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#coverageTarget","#readReplicaStateVersion","#latestBackup","#checkFirstBackupCovers","#firstBackupResolved"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debuggin; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n  #firstBackupResolved = false;\n  /**\n   * The replica's `stateVersion` when the monitor started, i.e. the point the\n   * backup has to reach before it covers what this task is about to serve.\n   * `undefined` when the replica could not be read, which degrades the gate to\n   * \"any watermark\" rather than hanging startup forever.\n   */\n  #coverageTarget: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n    this.#coverageTarget = this.#readReplicaStateVersion();\n\n    for await (const backedUp of this.#watermarks) {\n      if (this.#latestBackup) {\n        if (backedUp.watermark < this.#latestBackup.watermark) {\n          this.#lc.warn?.(`ignoring earlier backup watermark`, {backedUp});\n          continue;\n        }\n        if (backedUp.watermark === this.#latestBackup.watermark) {\n          this.#lc.debug?.(`ignoring redundant backup watermark`, {backedUp});\n          continue;\n        }\n      }\n      this.#lc.info?.(`received backup watermark`, {backedUp});\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n      this.#checkFirstBackupCovers(backedUp);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  /**\n   * Resolves {@link firstBackupReceived} once the backup actually covers the\n   * replica, rather than on the first watermark of any value.\n   *\n   * A new backup destination starts empty and is filled by re-uploading the\n   * local LTX chain, so the first watermarks read back out of it are real but\n   * far behind the replica. Releasing the readiness gate on one of those lets\n   * the task serve against a backup that does not yet cover it, which is what\n   * demotes a restoring view-syncer to Postgres catchup.\n   *\n   * The target is pinned at startup rather than compared against the replica's\n   * current position, which keeps the gate satisfiable under sustained writes.\n   */\n  #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n    if (this.#firstBackupResolved) {\n      return;\n    }\n    const target = this.#coverageTarget;\n    if (target !== undefined && backedUp.watermark < target) {\n      this.#lc.info?.(`backup does not yet cover the replica`, {\n        backedUp,\n        coverageTarget: target,\n      });\n      return;\n    }\n    this.#firstBackupResolved = true;\n    this.#firstBackupReceived.resolve();\n  }\n\n  #readReplicaStateVersion(): string | undefined {\n    let db;\n    try {\n      db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      const {stateVersion} = db\n        .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n        .get<{stateVersion: string}>();\n      return stateVersion;\n    } catch (e) {\n      // Without a target the gate degrades to its previous behavior. That is\n      // strictly better than blocking startup on a replica we cannot read.\n      this.#lc.warn?.(\n        `unable to read the replica's stateVersion; ` +\n          `the initial backup gate will accept the first watermark`,\n        e,\n      );\n      return undefined;\n    } finally {\n      db?.close();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAKC,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAKJ,IAAI,OAAO,yBAAyB;EACzC,KAAKK,qBAAqB;EAC1B,KAAKC,kBAAkB,KAAKC,yBAAyB;EAErD,WAAW,MAAM,YAAY,KAAKN,aAAa;GAC7C,IAAI,KAAKO,eAAe;IACtB,IAAI,SAAS,YAAY,KAAKA,cAAc,WAAW;KACrD,KAAKR,IAAI,OAAO,qCAAqC,EAAC,SAAQ,CAAC;KAC/D;IACF;IACA,IAAI,SAAS,cAAc,KAAKQ,cAAc,WAAW;KACvD,KAAKR,IAAI,QAAQ,uCAAuC,EAAC,SAAQ,CAAC;KAClE;IACF;GACF;GACA,KAAKA,IAAI,OAAO,6BAA6B,EAAC,SAAQ,CAAC;GACvD,KAAKQ,gBAAgB;GACrB,KAAKN,gBAAgB,qBAAqB,SAAS,SAAS;GAC5D,KAAKO,wBAAwB,QAAQ;EACvC;EACA,KAAKT,IAAI,OAAO,iDAAiD;CACnE;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAKU,sBACP;EAEF,MAAM,SAAS,KAAKJ;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAKN,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAKU,uBAAuB;EAC5B,KAAKN,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAKJ,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,EACnE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAKH,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAKC,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,eAAe,KAAKO;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAKR,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKH,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts

@@ -1,7 +1,13 @@
 import type { LogContext } from '@rocicorp/logger';
+import { type PreSerialized } from '../../types/streams.ts';
 import type { WatermarkedChange } from './change-streamer.ts';
 import type { Subscriber } from './subscriber.ts';
 export type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';
+export type PreSerializedBatch = PreSerialized & {
+    readonly changes: readonly WatermarkedChange[];
+};
+export declare function isPreSerializedBatch(val: unknown): val is PreSerializedBatch;
+export declare function preSerializeBatch(changes: readonly WatermarkedChange[]): PreSerializedBatch;
 /**
  * Enables event-driven early release of a {@link Broadcast}. When provided and a
  * majority has acked, the broadcast schedules its own consensus-timeout release
@@ -78,12 +84,12 @@ export declare class Broadcast {
      * Sends the change to the subscribers without the tracking machinery.
      * This is suitable for fire-and-forget (i.e. pipelined) sends.
      */
-    static withoutTracking(subscribers: Iterable<Subscriber>, change: WatermarkedChange): void;
+    static withoutTracking(subscribers: Iterable<Subscriber>, change: WatermarkedChange | readonly WatermarkedChange[]): void;
     /**
      * Broadcasts the `change` to the `subscribers` and tracks their
      * completion.
      */
-    constructor(lc: LogContext, subscribers: Iterable<Subscriber>, change: WatermarkedChange, earlyRelease?: EarlyReleaseOptions);
+    constructor(lc: LogContext, subscribers: Iterable<Subscriber>, change: WatermarkedChange | readonly WatermarkedChange[], earlyRelease?: EarlyReleaseOptions);
     get isDone(): boolean;
     get done(): Promise<void>;
     get releaseMode(): BroadcastReleaseMode;

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E;;;;;GAKG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,2EAA2E;IAC3E,0BAA0B,EAAE,MAAM,CAAC;IACnC,mDAAmD;IACnD,YAAY,CAAC,EAAE,OAAO,UAAU,CAAC;IACjC,gEAAgE;IAChE,cAAc,CAAC,EAAE,OAAO,YAAY,CAAC;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB,QAK1B;IA0BD;;;OAGG;IACH,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB,EACzB,YAAY,CAAC,EAAE,mBAAmB,EA6BnC;IAwDD,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED,eAAe,CAAC,GAAG,EAAE,MAAM,QAa1B;CAiBF"}
+{"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAkB,KAAK,aAAa,EAAC,MAAM,wBAAwB,CAAC;AAC3E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG;IAC/C,QAAQ,CAAC,OAAO,EAAE,SAAS,iBAAiB,EAAE,CAAC;CAChD,CAAC;AAEF,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,OAAO,GAAG,GAAG,IAAI,kBAAkB,CAE5E;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,SAAS,iBAAiB,EAAE,GACpC,kBAAkB,CAYpB;AAED;;;;;GAKG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,2EAA2E;IAC3E,0BAA0B,EAAE,MAAM,CAAC;IACnC,mDAAmD;IACnD,YAAY,CAAC,EAAE,OAAO,UAAU,CAAC;IACjC,gEAAgE;IAChE,cAAc,CAAC,EAAE,OAAO,YAAY,CAAC;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB,GAAG,SAAS,iBAAiB,EAAE,QAOzD;IA0BD;;;OAGG;IACH,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB,GAAG,SAAS,iBAAiB,EAAE,EACxD,YAAY,CAAC,EAAE,mBAAmB,EA+BnC;IAwDD,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED,eAAe,CAAC,GAAG,EAAE,MAAM,QAa1B;CAiBF"}

package/out/zero-cache/src/services/change-streamer/broadcast.js

@@ -1,5 +1,17 @@
+import { must } from "../../../../shared/src/must.js";
+import "../../types/streams.js";
 import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-streamer/broadcast.ts
+function preSerializeBatch(changes) {
+	const jsonList = changes.map((c) => c[2]);
+	const joined = jsonList.join(",");
+	const payload = Buffer.from(changes.length === 1 ? `,"msg":${jsonList[0]}}` : `,"batch":[${joined}]}`, "utf8");
+	return {
+		changes,
+		payload,
+		byteLength: payload.length
+	};
+}
 /**
 * Initiates and tracks the progress of a change broadcasted to
 * a set of subscribers, implementing **consensus-based timeouts**
@@ -62,7 +74,9 @@ var Broadcast = class {
 	* This is suitable for fire-and-forget (i.e. pipelined) sends.
 	*/
 	static withoutTracking(subscribers, change) {
-		for (const sub of subscribers) sub.send(change);
+		const changes = toBatch(change);
+		const preSerialized = preSerializeBatch(changes);
+		for (const sub of subscribers) sub.sendBatch(changes, preSerialized);
 	}
 	#lc;
 	#pending;
@@ -83,18 +97,20 @@ var Broadcast = class {
 	* completion.
 	*/
 	constructor(lc, subscribers, change, earlyRelease) {
+		const changes = toBatch(change);
+		const preSerialized = preSerializeBatch(changes);
 		this.#lc = lc;
 		this.#pending = new Set(subscribers);
 		this.#completed = [];
-		this.#watermark = change[0];
+		this.#watermark = must(changes.at(-1))[0];
 		this.#majority = Math.floor(this.#pending.size / 2) + 1;
 		this.#earlyReleaseTimeoutProportion = earlyRelease?.consensusTimeoutProportion;
 		this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;
 		this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;
 		for (const sub of this.#pending) {
-			const changes = sub.numPending + 1;
+			const totalChanges = sub.numPending + changes.length;
 			if (sub.mode === "backup") this.#needsBackupResponse ||= !sub.isBacklogged();
-			sub.send(change).catch(() => {}).finally(() => this.#markCompleted(sub, changes));
+			sub.sendBatch(changes, preSerialized).catch(() => {}).finally(() => this.#markCompleted(sub, totalChanges));
 		}
 		if (this.#pending.size === 0) this.#setDone("all-subscribers");
 	}
@@ -161,6 +177,10 @@ var Broadcast = class {
 		});
 	}
 };
+function toBatch(change) {
+	if (change.length === 0) return [];
+	return Array.isArray(change[0]) ? change : [change];
+}
 //#endregion
 export { Broadcast };
 

package/out/zero-cache/src/services/change-streamer/broadcast.js.map

@@ -1 +1 @@
-{"version":3,"file":"broadcast.js","names":["#lc","#pending","#completed","#done","#watermark","#majority","#start","#earlyReleaseTimeoutProportion","#setTimeout","#clearTimeout","#needsBackupResponse","#markCompleted","#setDone","#isDone","#earlyReleaseTimer","#logWithState","#releaseMode"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';\n\n/**\n * Enables event-driven early release of a {@link Broadcast}. When provided and a\n * majority has acked, the broadcast schedules its own consensus-timeout release\n * proportional to how long the majority took to ack, instead of relying on the\n * Forwarder's periodic progress tick.\n */\nexport type EarlyReleaseOptions = {\n  /** Time to wait, proportional to majority completion, before releasing. */\n  consensusTimeoutProportion: number;\n  /** Injectable timer, for deterministic testing. */\n  setTimeoutFn?: typeof setTimeout;\n  /** Injectable timer cancellation, for deterministic testing. */\n  clearTimeoutFn?: typeof clearTimeout;\n};\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers, implementing **consensus-based timeouts**\n * to effect short-term graceful degradation when a minority of\n * subscribers are behind.\n *\n * ### Background\n *\n * The purpose of flow control is to pull upstream replication changes\n * no faster than the rate they are processed by downstream subscribers\n * in the steady state. In the change-streamer, this is done by occasionally\n * waiting for ACKs from subscribers before continuing; without doing so,\n * I/O buffers fill up and cause the system to spend most of its time in GC.\n *\n * However, the naive algorithm of always waiting for all subscribers (e.g.\n * `Promise.all()`) can behave poorly in scenarios where subscribers\n * are imbalanced:\n * * New subscribers may have a backlog of changes to catch up with.\n *   Having all subscribers wait for the new subscriber to catch up results\n *   in delaying the entire application.\n * * Broken TCP connections similarly require all subscribers to wait until\n *   connection liveness checks kick in and disconnect the subscriber.\n *\n * A simplistic approach is to add a limit to the amount of time waiting for\n * subscribers, i.e. an ack timeout. However, deciding what this timeout\n * should be is non-trivial because of the heterogeneous nature of changes;\n * while most changes operate on single rows and are relatively predictable\n * in terms of running time, some changes are table-wide operations and can\n * legitimately take an arbitrary amount of time. In such scenarios, a\n * timeout that is too short can stop progress on replication altogether.\n *\n * ### Consensus-based Timeout Algorithm\n *\n * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n * used:\n * * Wait for more than half of the subscribers to finish. (In\n *   case of a single node, or the case of one replication-manager\n *   and one view-syncer, this reduces to waiting for all subscribers.)\n * * Once more than half of the subscribers have finished, proceed after\n *   the configured timeout, proportional to the time it took for the\n *   majority of subscribers to ack, even if not all subscribers have\n *   finished.\n *\n * In other words, the subscribers themselves are used to determine the\n * timeout of each batch of changes; the majority determines this when\n * they complete, upon which a timeout is logically started.\n *\n * In the common case, the remaining subscribers finish soon afterward and\n * the timeout never elapses. However, in pathological cases where a minority\n * of subscribers have a disproportionate amount of load, some will still\n * be processing (or otherwise unresponsive). These subscribers are given\n * a bounded amount of time to catch up at each flushed batch, up to the\n * timeout interval. This guarantees eventual catchup because the\n * subscribers with a backlog of changes necessarily have a higher\n * processing rate than the subscribers that finished (and are made to wait).\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n  ) {\n    for (const sub of subscribers) {\n      void sub.send(change);\n    }\n  }\n\n  readonly #lc: LogContext;\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n  #releaseMode: BroadcastReleaseMode | undefined;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n\n  // When set, the broadcast schedules its own consensus-timeout release instead\n  // of waiting for the Forwarder's periodic progress tick.\n  readonly #earlyReleaseTimeoutProportion: number | undefined;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  #earlyReleaseTimer: ReturnType<typeof setTimeout> | undefined;\n\n  // Tracks whether the backup-replicator responded. It is a required member\n  // of the consensus timeout since it cannot be dropped as a laggard (since\n  // that would shutdown the replication-manager itself).\n  #needsBackupResponse = false;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(\n    lc: LogContext,\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange,\n    earlyRelease?: EarlyReleaseOptions,\n  ) {\n    this.#lc = lc;\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = change[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n    this.#earlyReleaseTimeoutProportion =\n      earlyRelease?.consensusTimeoutProportion;\n    this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;\n\n    for (const sub of this.#pending) {\n      const changes = sub.numPending + 1; // add one for this `change`\n      if (sub.mode === 'backup') {\n        // Only gate consensus on the backup-replicator after it has finished\n        // its initial catchup.\n        this.#needsBackupResponse ||= !sub.isBacklogged();\n      }\n      void sub\n        .send(change)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, changes));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    if (this.#isDone) {\n      return;\n    }\n    const elapsed = performance.now() - this.#start;\n    sub.trackResponseResult('on-time');\n    if (sub.mode === 'backup') {\n      this.#needsBackupResponse = false;\n    }\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n      return;\n    }\n    // Event-driven consensus-timeout: the moment a majority acks, and it\n    // includes the backup-replicator, arm a single release timer\n    // proportional to how long the majority took to ack. The broadcast\n    // resolves when it fires, unless all subscribers ack first.\n    if (\n      this.#earlyReleaseTimeoutProportion !== undefined &&\n      this.#earlyReleaseTimer === undefined &&\n      this.#completed.length >= this.#majority &&\n      !this.#needsBackupResponse\n    ) {\n      const timeoutMs = Math.ceil(\n        elapsed * this.#earlyReleaseTimeoutProportion,\n      );\n      this.#earlyReleaseTimer = this.#setTimeout(\n        () => this.#setDone('consensus-timeout'),\n        timeoutMs,\n      );\n    }\n  }\n\n  #setDone(releaseMode: BroadcastReleaseMode) {\n    if (this.#isDone) {\n      return;\n    }\n    if (releaseMode === 'consensus-timeout') {\n      const elapsed = performance.now() - this.#start;\n      this.#logWithState(\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n    }\n    this.#pending.forEach(sub => sub.trackResponseResult('timed-out'));\n    this.#isDone = true;\n    this.#releaseMode = releaseMode;\n    this.#clearTimeout(this.#earlyReleaseTimer);\n    this.#earlyReleaseTimer = undefined;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  get releaseMode(): BroadcastReleaseMode {\n    return this.#releaseMode ?? 'all-subscribers';\n  }\n\n  logSlowRequests(now: number) {\n    if (this.#isDone || this.#pending.size === 0) {\n      return;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n    }\n  }\n\n  #logWithState(msg: string, elapsed: number) {\n    this.#lc\n      .withContext('watermark', this.#watermark)\n      .info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      });\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8EA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,KAAK,MAAM,OAAO,aAChB,IAAS,KAAK,MAAM;CAExB;CAEA;CACA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CACV;CAEA;CACA;CAEA,SAAkB,YAAY,IAAI;CAIlC;CACA;CACA;CACA;CAKA,uBAAuB;;;;;CAMvB,YACE,IACA,aACA,QACA,cACA;EACA,KAAKA,MAAM;EACX,KAAKC,WAAW,IAAI,IAAI,WAAW;EACnC,KAAKC,aAAa,CAAC;EACnB,KAAKE,aAAa,OAAO;EACzB,KAAKC,YAAY,KAAK,MAAM,KAAKJ,SAAS,OAAO,CAAC,IAAI;EACtD,KAAKM,iCACH,cAAc;EAChB,KAAKC,cAAc,cAAc,gBAAgB;EACjD,KAAKC,gBAAgB,cAAc,kBAAkB;EAErD,KAAK,MAAM,OAAO,KAAKR,UAAU;GAC/B,MAAM,UAAU,IAAI,aAAa;GACjC,IAAI,IAAI,SAAS,UAGf,KAAKS,yBAAyB,CAAC,IAAI,aAAa;GAElD,IACG,KAAK,MAAM,EACX,YAAY,CAAC,CAAC,EACd,cAAc,KAAKC,eAAe,KAAK,OAAO,CAAC;EACpD;EAGA,IAAI,KAAKV,SAAS,SAAS,GACzB,KAAKW,SAAS,iBAAiB;CAEnC;CAEA,eAAe,KAAiB,SAAiB;EAC/C,IAAI,KAAKC,SACP;EAEF,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKP;EACzC,IAAI,oBAAoB,SAAS;EACjC,IAAI,IAAI,SAAS,UACf,KAAKI,uBAAuB;EAE9B,KAAKR,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAKD,SAAS,OAAO,GAAG;EACxB,IAAI,KAAKA,SAAS,SAAS,GAAG;GAC5B,KAAKW,SAAS,iBAAiB;GAC/B;EACF;EAKA,IACE,KAAKL,mCAAmC,KAAA,KACxC,KAAKO,uBAAuB,KAAA,KAC5B,KAAKZ,WAAW,UAAU,KAAKG,aAC/B,CAAC,KAAKK,sBACN;GACA,MAAM,YAAY,KAAK,KACrB,UAAU,KAAKH,8BACjB;GACA,KAAKO,qBAAqB,KAAKN,kBACvB,KAAKI,SAAS,mBAAmB,GACvC,SACF;EACF;CACF;CAEA,SAAS,aAAmC;EAC1C,IAAI,KAAKC,SACP;EAEF,IAAI,gBAAgB,qBAAqB;GACvC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKP;GACzC,KAAKS,cACH,mBAAmB,KAAKd,SAAS,KAAK,+BACtC,OACF;EACF;EACA,KAAKA,SAAS,SAAQ,QAAO,IAAI,oBAAoB,WAAW,CAAC;EACjE,KAAKY,UAAU;EACf,KAAKG,eAAe;EACpB,KAAKP,cAAc,KAAKK,kBAAkB;EAC1C,KAAKA,qBAAqB,KAAA;EAC1B,KAAKX,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAKU;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAKV,MAAM;CACpB;CAEA,IAAI,cAAoC;EACtC,OAAO,KAAKa,gBAAgB;CAC9B;CAEA,gBAAgB,KAAa;EAC3B,IAAI,KAAKH,WAAW,KAAKZ,SAAS,SAAS,GACzC;EAEF,MAAM,UAAU,MAAM,KAAKK;EAC3B,IAAI,KAAKJ,WAAW,SAAS,KAAKG;OAC5B,WAAW,KACb,KAAKU,cACH,wBAAwB,KAAKV,UAAU,yBACvC,OACF;EAAA;CAGN;CAEA,cAAc,KAAa,SAAiB;EAC1C,KAAKL,IACF,YAAY,aAAa,KAAKI,UAAU,EACxC,OAAO,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAAO;GAC3C,WAAW,KAAKF,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAKD,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CAAC;CACL;AACF"}
+{"version":3,"file":"broadcast.js","names":["#lc","#pending","#completed","#done","#watermark","#majority","#start","#earlyReleaseTimeoutProportion","#setTimeout","#clearTimeout","#needsBackupResponse","#markCompleted","#setDone","#isDone","#earlyReleaseTimer","#logWithState","#releaseMode"],"sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {isPreSerialized, type PreSerialized} from '../../types/streams.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';\n\nexport type PreSerializedBatch = PreSerialized & {\n  readonly changes: readonly WatermarkedChange[];\n};\n\nexport function isPreSerializedBatch(val: unknown): val is PreSerializedBatch {\n  return isPreSerialized(val) && 'changes' in val && Array.isArray(val.changes);\n}\n\nexport function preSerializeBatch(\n  changes: readonly WatermarkedChange[],\n): PreSerializedBatch {\n  const jsonList = changes.map(c => c[2]);\n  const joined = jsonList.join(',');\n  const payload = Buffer.from(\n    changes.length === 1 ? `,\"msg\":${jsonList[0]}}` : `,\"batch\":[${joined}]}`,\n    'utf8',\n  );\n  return {\n    changes,\n    payload,\n    byteLength: payload.length,\n  };\n}\n\n/**\n * Enables event-driven early release of a {@link Broadcast}. When provided and a\n * majority has acked, the broadcast schedules its own consensus-timeout release\n * proportional to how long the majority took to ack, instead of relying on the\n * Forwarder's periodic progress tick.\n */\nexport type EarlyReleaseOptions = {\n  /** Time to wait, proportional to majority completion, before releasing. */\n  consensusTimeoutProportion: number;\n  /** Injectable timer, for deterministic testing. */\n  setTimeoutFn?: typeof setTimeout;\n  /** Injectable timer cancellation, for deterministic testing. */\n  clearTimeoutFn?: typeof clearTimeout;\n};\n\n/**\n * Initiates and tracks the progress of a change broadcasted to\n * a set of subscribers, implementing **consensus-based timeouts**\n * to effect short-term graceful degradation when a minority of\n * subscribers are behind.\n *\n * ### Background\n *\n * The purpose of flow control is to pull upstream replication changes\n * no faster than the rate they are processed by downstream subscribers\n * in the steady state. In the change-streamer, this is done by occasionally\n * waiting for ACKs from subscribers before continuing; without doing so,\n * I/O buffers fill up and cause the system to spend most of its time in GC.\n *\n * However, the naive algorithm of always waiting for all subscribers (e.g.\n * `Promise.all()`) can behave poorly in scenarios where subscribers\n * are imbalanced:\n * * New subscribers may have a backlog of changes to catch up with.\n *   Having all subscribers wait for the new subscriber to catch up results\n *   in delaying the entire application.\n * * Broken TCP connections similarly require all subscribers to wait until\n *   connection liveness checks kick in and disconnect the subscriber.\n *\n * A simplistic approach is to add a limit to the amount of time waiting for\n * subscribers, i.e. an ack timeout. However, deciding what this timeout\n * should be is non-trivial because of the heterogeneous nature of changes;\n * while most changes operate on single rows and are relatively predictable\n * in terms of running time, some changes are table-wide operations and can\n * legitimately take an arbitrary amount of time. In such scenarios, a\n * timeout that is too short can stop progress on replication altogether.\n *\n * ### Consensus-based Timeout Algorithm\n *\n * To address these shortcomings, a \"consensus-based timeout\" algorithm is\n * used:\n * * Wait for more than half of the subscribers to finish. (In\n *   case of a single node, or the case of one replication-manager\n *   and one view-syncer, this reduces to waiting for all subscribers.)\n * * Once more than half of the subscribers have finished, proceed after\n *   the configured timeout, proportional to the time it took for the\n *   majority of subscribers to ack, even if not all subscribers have\n *   finished.\n *\n * In other words, the subscribers themselves are used to determine the\n * timeout of each batch of changes; the majority determines this when\n * they complete, upon which a timeout is logically started.\n *\n * In the common case, the remaining subscribers finish soon afterward and\n * the timeout never elapses. However, in pathological cases where a minority\n * of subscribers have a disproportionate amount of load, some will still\n * be processing (or otherwise unresponsive). These subscribers are given\n * a bounded amount of time to catch up at each flushed batch, up to the\n * timeout interval. This guarantees eventual catchup because the\n * subscribers with a backlog of changes necessarily have a higher\n * processing rate than the subscribers that finished (and are made to wait).\n */\nexport class Broadcast {\n  /**\n   * Sends the change to the subscribers without the tracking machinery.\n   * This is suitable for fire-and-forget (i.e. pipelined) sends.\n   */\n  static withoutTracking(\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange | readonly WatermarkedChange[],\n  ) {\n    const changes = toBatch(change);\n    const preSerialized = preSerializeBatch(changes);\n    for (const sub of subscribers) {\n      void sub.sendBatch(changes, preSerialized);\n    }\n  }\n\n  readonly #lc: LogContext;\n  readonly #pending: Set<Subscriber>;\n  readonly #completed: Completed[];\n  readonly #done = resolver();\n  #isDone = false;\n  #releaseMode: BroadcastReleaseMode | undefined;\n\n  readonly #watermark: string;\n  readonly #majority: number;\n\n  readonly #start = performance.now();\n\n  // When set, the broadcast schedules its own consensus-timeout release instead\n  // of waiting for the Forwarder's periodic progress tick.\n  readonly #earlyReleaseTimeoutProportion: number | undefined;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  #earlyReleaseTimer: ReturnType<typeof setTimeout> | undefined;\n\n  // Tracks whether the backup-replicator responded. It is a required member\n  // of the consensus timeout since it cannot be dropped as a laggard (since\n  // that would shutdown the replication-manager itself).\n  #needsBackupResponse = false;\n\n  /**\n   * Broadcasts the `change` to the `subscribers` and tracks their\n   * completion.\n   */\n  constructor(\n    lc: LogContext,\n    subscribers: Iterable<Subscriber>,\n    change: WatermarkedChange | readonly WatermarkedChange[],\n    earlyRelease?: EarlyReleaseOptions,\n  ) {\n    const changes = toBatch(change);\n    const preSerialized = preSerializeBatch(changes);\n    this.#lc = lc;\n    this.#pending = new Set(subscribers);\n    this.#completed = [];\n    this.#watermark = must(changes.at(-1))[0];\n    this.#majority = Math.floor(this.#pending.size / 2) + 1;\n    this.#earlyReleaseTimeoutProportion =\n      earlyRelease?.consensusTimeoutProportion;\n    this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;\n\n    for (const sub of this.#pending) {\n      const totalChanges = sub.numPending + changes.length;\n      if (sub.mode === 'backup') {\n        // Only gate consensus on the backup-replicator after it has finished\n        // its initial catchup.\n        this.#needsBackupResponse ||= !sub.isBacklogged();\n      }\n      void sub\n        .sendBatch(changes, preSerialized)\n        .catch(() => {})\n        .finally(() => this.#markCompleted(sub, totalChanges));\n    }\n\n    // set done if there are no subscribers (mainly for tests)\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n    }\n  }\n\n  #markCompleted(sub: Subscriber, changes: number) {\n    if (this.#isDone) {\n      return;\n    }\n    const elapsed = performance.now() - this.#start;\n    sub.trackResponseResult('on-time');\n    if (sub.mode === 'backup') {\n      this.#needsBackupResponse = false;\n    }\n    this.#completed.push({sub, changes, elapsed});\n    this.#pending.delete(sub);\n    if (this.#pending.size === 0) {\n      this.#setDone('all-subscribers');\n      return;\n    }\n    // Event-driven consensus-timeout: the moment a majority acks, and it\n    // includes the backup-replicator, arm a single release timer\n    // proportional to how long the majority took to ack. The broadcast\n    // resolves when it fires, unless all subscribers ack first.\n    if (\n      this.#earlyReleaseTimeoutProportion !== undefined &&\n      this.#earlyReleaseTimer === undefined &&\n      this.#completed.length >= this.#majority &&\n      !this.#needsBackupResponse\n    ) {\n      const timeoutMs = Math.ceil(\n        elapsed * this.#earlyReleaseTimeoutProportion,\n      );\n      this.#earlyReleaseTimer = this.#setTimeout(\n        () => this.#setDone('consensus-timeout'),\n        timeoutMs,\n      );\n    }\n  }\n\n  #setDone(releaseMode: BroadcastReleaseMode) {\n    if (this.#isDone) {\n      return;\n    }\n    if (releaseMode === 'consensus-timeout') {\n      const elapsed = performance.now() - this.#start;\n      this.#logWithState(\n        `continuing with ${this.#pending.size} subscriber(s) still pending`,\n        elapsed,\n      );\n    }\n    this.#pending.forEach(sub => sub.trackResponseResult('timed-out'));\n    this.#isDone = true;\n    this.#releaseMode = releaseMode;\n    this.#clearTimeout(this.#earlyReleaseTimer);\n    this.#earlyReleaseTimer = undefined;\n    this.#done.resolve();\n  }\n\n  get isDone(): boolean {\n    return this.#isDone;\n  }\n\n  get done(): Promise<void> {\n    return this.#done.promise;\n  }\n\n  get releaseMode(): BroadcastReleaseMode {\n    return this.#releaseMode ?? 'all-subscribers';\n  }\n\n  logSlowRequests(now: number) {\n    if (this.#isDone || this.#pending.size === 0) {\n      return;\n    }\n    const elapsed = now - this.#start;\n    if (this.#completed.length < this.#majority) {\n      if (elapsed >= 1000) {\n        this.#logWithState(\n          `waiting for at least ${this.#majority} subscribers to finish`,\n          elapsed,\n        );\n      }\n    }\n  }\n\n  #logWithState(msg: string, elapsed: number) {\n    this.#lc\n      .withContext('watermark', this.#watermark)\n      .info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {\n        completed: this.#completed.map(d => ({\n          id: d.sub.id,\n          processed: d.changes,\n          elapsed: d.elapsed,\n        })),\n        pending: Array.from(this.#pending, sub => ({\n          id: sub.id,\n          ...sub.getStats(),\n        })),\n      });\n  }\n}\n\n/** Tracks the completed result of a single subscriber. */\ntype Completed = {\n  sub: Subscriber;\n  /** The number of changes processed. */\n  changes: number;\n  /** The elapsed milliseconds. */\n  elapsed: number;\n};\n\nfunction toBatch(\n  change: WatermarkedChange | readonly WatermarkedChange[],\n): readonly WatermarkedChange[] {\n  if (change.length === 0) {\n    return [];\n  }\n  return Array.isArray(change[0])\n    ? (change as readonly WatermarkedChange[])\n    : [change as WatermarkedChange];\n}\n"],"mappings":";;;;AAiBA,SAAgB,kBACd,SACoB;CACpB,MAAM,WAAW,QAAQ,KAAI,MAAK,EAAE,EAAE;CACtC,MAAM,SAAS,SAAS,KAAK,GAAG;CAChC,MAAM,UAAU,OAAO,KACrB,QAAQ,WAAW,IAAI,UAAU,SAAS,GAAG,KAAK,aAAa,OAAO,KACtE,MACF;CACA,OAAO;EACL;EACA;EACA,YAAY,QAAQ;CACtB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,IAAa,YAAb,MAAuB;;;;;CAKrB,OAAO,gBACL,aACA,QACA;EACA,MAAM,UAAU,QAAQ,MAAM;EAC9B,MAAM,gBAAgB,kBAAkB,OAAO;EAC/C,KAAK,MAAM,OAAO,aAChB,IAAS,UAAU,SAAS,aAAa;CAE7C;CAEA;CACA;CACA;CACA,QAAiB,SAAS;CAC1B,UAAU;CACV;CAEA;CACA;CAEA,SAAkB,YAAY,IAAI;CAIlC;CACA;CACA;CACA;CAKA,uBAAuB;;;;;CAMvB,YACE,IACA,aACA,QACA,cACA;EACA,MAAM,UAAU,QAAQ,MAAM;EAC9B,MAAM,gBAAgB,kBAAkB,OAAO;EAC/C,KAAKA,MAAM;EACX,KAAKC,WAAW,IAAI,IAAI,WAAW;EACnC,KAAKC,aAAa,CAAC;EACnB,KAAKE,aAAa,KAAK,QAAQ,GAAG,EAAE,CAAC,EAAE;EACvC,KAAKC,YAAY,KAAK,MAAM,KAAKJ,SAAS,OAAO,CAAC,IAAI;EACtD,KAAKM,iCACH,cAAc;EAChB,KAAKC,cAAc,cAAc,gBAAgB;EACjD,KAAKC,gBAAgB,cAAc,kBAAkB;EAErD,KAAK,MAAM,OAAO,KAAKR,UAAU;GAC/B,MAAM,eAAe,IAAI,aAAa,QAAQ;GAC9C,IAAI,IAAI,SAAS,UAGf,KAAKS,yBAAyB,CAAC,IAAI,aAAa;GAElD,IACG,UAAU,SAAS,aAAa,EAChC,YAAY,CAAC,CAAC,EACd,cAAc,KAAKC,eAAe,KAAK,YAAY,CAAC;EACzD;EAGA,IAAI,KAAKV,SAAS,SAAS,GACzB,KAAKW,SAAS,iBAAiB;CAEnC;CAEA,eAAe,KAAiB,SAAiB;EAC/C,IAAI,KAAKC,SACP;EAEF,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKP;EACzC,IAAI,oBAAoB,SAAS;EACjC,IAAI,IAAI,SAAS,UACf,KAAKI,uBAAuB;EAE9B,KAAKR,WAAW,KAAK;GAAC;GAAK;GAAS;EAAO,CAAC;EAC5C,KAAKD,SAAS,OAAO,GAAG;EACxB,IAAI,KAAKA,SAAS,SAAS,GAAG;GAC5B,KAAKW,SAAS,iBAAiB;GAC/B;EACF;EAKA,IACE,KAAKL,mCAAmC,KAAA,KACxC,KAAKO,uBAAuB,KAAA,KAC5B,KAAKZ,WAAW,UAAU,KAAKG,aAC/B,CAAC,KAAKK,sBACN;GACA,MAAM,YAAY,KAAK,KACrB,UAAU,KAAKH,8BACjB;GACA,KAAKO,qBAAqB,KAAKN,kBACvB,KAAKI,SAAS,mBAAmB,GACvC,SACF;EACF;CACF;CAEA,SAAS,aAAmC;EAC1C,IAAI,KAAKC,SACP;EAEF,IAAI,gBAAgB,qBAAqB;GACvC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKP;GACzC,KAAKS,cACH,mBAAmB,KAAKd,SAAS,KAAK,+BACtC,OACF;EACF;EACA,KAAKA,SAAS,SAAQ,QAAO,IAAI,oBAAoB,WAAW,CAAC;EACjE,KAAKY,UAAU;EACf,KAAKG,eAAe;EACpB,KAAKP,cAAc,KAAKK,kBAAkB;EAC1C,KAAKA,qBAAqB,KAAA;EAC1B,KAAKX,MAAM,QAAQ;CACrB;CAEA,IAAI,SAAkB;EACpB,OAAO,KAAKU;CACd;CAEA,IAAI,OAAsB;EACxB,OAAO,KAAKV,MAAM;CACpB;CAEA,IAAI,cAAoC;EACtC,OAAO,KAAKa,gBAAgB;CAC9B;CAEA,gBAAgB,KAAa;EAC3B,IAAI,KAAKH,WAAW,KAAKZ,SAAS,SAAS,GACzC;EAEF,MAAM,UAAU,MAAM,KAAKK;EAC3B,IAAI,KAAKJ,WAAW,SAAS,KAAKG;OAC5B,WAAW,KACb,KAAKU,cACH,wBAAwB,KAAKV,UAAU,yBACvC,OACF;EAAA;CAGN;CAEA,cAAc,KAAa,SAAiB;EAC1C,KAAKL,IACF,YAAY,aAAa,KAAKI,UAAU,EACxC,OAAO,GAAG,IAAI,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAAO;GAC3C,WAAW,KAAKF,WAAW,KAAI,OAAM;IACnC,IAAI,EAAE,IAAI;IACV,WAAW,EAAE;IACb,SAAS,EAAE;GACb,EAAE;GACF,SAAS,MAAM,KAAK,KAAKD,WAAU,SAAQ;IACzC,IAAI,IAAI;IACR,GAAG,IAAI,SAAS;GAClB,EAAE;EACJ,CAAC;CACL;AACF;AAWA,SAAS,QACP,QAC8B;CAC9B,IAAI,OAAO,WAAW,GACpB,OAAO,CAAC;CAEV,OAAO,MAAM,QAAQ,OAAO,EAAE,IACzB,SACD,CAAC,MAA2B;AAClC"}

package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts

@@ -5,13 +5,28 @@ export type ChangeLogEntry = {
     tag: string;
     change: string;
 };
+export type SerializedChangeStreamData = {
+    /** The canonical downstream representation forwarded to subscribers. */
+    readonly json: string;
+    /** The already-serialized change tuple element stored in change logs. */
+    readonly change: string;
+};
+/**
+ * Serializes a data-plane message and its stored change payload in one pass.
+ *
+ * Change logs retain only the second tuple element for backwards
+ * compatibility. Serializing that element first avoids stringifying the full
+ * message and then copying the payload back out of the resulting string on the
+ * replication hot path.
+ */
+export declare function serializeChangeStreamDataWithChange(data: ChangeStreamData): SerializedChangeStreamData;
 /** Serializes a data-plane message to the canonical downstream JSON form. */
 export declare function serializeChangeStreamData(data: ChangeStreamData): string;
 /**
- * Extracts the stringified change message from the stringified stream message
- * (the second tuple element). This allows the stream message to be stringified
- * once while storing only the change message in the change log for backwards
- * compatibility.
+ * Extracts the second tuple element for callers that only have the canonical
+ * downstream JSON. The live write path uses
+ * {@link serializeChangeStreamDataWithChange} and does not scan or copy the
+ * completed message.
  */
 export declare function extractChangeSubstring(streamMessageJSON: string, tag: ChangeTag | undefined): string;
 /** Reconstructs the canonical downstream JSON around a stored change. */

package/out/zero-cache/src/services/change-streamer/change-log-codec.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-codec.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-codec.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,SAAS,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAEvE,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,6EAA6E;AAC7E,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAExE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,SAAS,GAAG,SAAS,GACzB,MAAM,CAkBR;AAED,yEAAyE;AACzE,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,cAAc,GACpB,iBAAiB,CAqBnB"}
+{"version":3,"file":"change-log-codec.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-codec.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,SAAS,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAEvE,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,wEAAwE;IACxE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,mCAAmC,CACjD,IAAI,EAAE,gBAAgB,GACrB,0BAA0B,CAc5B;AAED,6EAA6E;AAC7E,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAExE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,SAAS,GAAG,SAAS,GACzB,MAAM,CAkBR;AAED,yEAAyE;AACzE,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,cAAc,GACpB,iBAAiB,CAqBnB"}

package/out/zero-cache/src/services/change-streamer/change-log-codec.js

@@ -1,14 +1,34 @@
 import { BigIntJSON } from "../../../../shared/src/bigint-json.js";
 //#region ../zero-cache/src/services/change-streamer/change-log-codec.ts
-/** Serializes a data-plane message to the canonical downstream JSON form. */
-function serializeChangeStreamData(data) {
-	return BigIntJSON.stringify(data);
+/**
+* Serializes a data-plane message and its stored change payload in one pass.
+*
+* Change logs retain only the second tuple element for backwards
+* compatibility. Serializing that element first avoids stringifying the full
+* message and then copying the payload back out of the resulting string on the
+* replication hot path.
+*/
+function serializeChangeStreamDataWithChange(data) {
+	const change = BigIntJSON.stringify(data[1]);
+	const type = data[0];
+	switch (type) {
+		case "begin":
+		case "commit": return {
+			json: `[${JSON.stringify(type)},${change},${BigIntJSON.stringify(data[2])}]`,
+			change
+		};
+		case "data":
+		case "rollback": return {
+			json: `[${JSON.stringify(type)},${change}]`,
+			change
+		};
+	}
 }
 /**
-* Extracts the stringified change message from the stringified stream message
-* (the second tuple element). This allows the stream message to be stringified
-* once while storing only the change message in the change log for backwards
-* compatibility.
+* Extracts the second tuple element for callers that only have the canonical
+* downstream JSON. The live write path uses
+* {@link serializeChangeStreamDataWithChange} and does not scan or copy the
+* completed message.
 */
 function extractChangeSubstring(streamMessageJSON, tag) {
 	switch (tag) {
@@ -45,6 +65,6 @@ function reconstructWatermarkedChange(entry) {
 	}
 }
 //#endregion
-export { extractChangeSubstring, reconstructWatermarkedChange, serializeChangeStreamData };
+export { extractChangeSubstring, reconstructWatermarkedChange, serializeChangeStreamDataWithChange };
 
 //# sourceMappingURL=change-log-codec.js.map

package/out/zero-cache/src/services/change-streamer/change-log-codec.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-codec.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-codec.ts"],"sourcesContent":["import {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer.ts';\n\nexport type ChangeLogEntry = {\n  watermark: string;\n  tag: string;\n  change: string;\n};\n\n/** Serializes a data-plane message to the canonical downstream JSON form. */\nexport function serializeChangeStreamData(data: ChangeStreamData): string {\n  return BigIntJSON.stringify(data);\n}\n\n/**\n * Extracts the stringified change message from the stringified stream message\n * (the second tuple element). This allows the stream message to be stringified\n * once while storing only the change message in the change log for backwards\n * compatibility.\n */\nexport function extractChangeSubstring(\n  streamMessageJSON: string,\n  tag: ChangeTag | undefined,\n): string {\n  switch (tag) {\n    case 'begin':\n    case 'commit':\n      // e.g.\n      // [\"begin\",<message-json>,{\"commitWatermark\":\"92fj2d0s\"}]\n      // [\"commit\",<message-json>,{\"watermark\":\"92fj2d0s\"}]\n      return streamMessageJSON.substring(\n        streamMessageJSON.indexOf(',') + 1,\n        streamMessageJSON.lastIndexOf(','),\n      );\n    default:\n      // [\"data\",<message-json>]\n      return streamMessageJSON.substring(\n        streamMessageJSON.indexOf(',') + 1,\n        streamMessageJSON.lastIndexOf(']'),\n      );\n  }\n}\n\n/** Reconstructs the canonical downstream JSON around a stored change. */\nexport function reconstructWatermarkedChange(\n  entry: ChangeLogEntry,\n): WatermarkedChange {\n  const {watermark, change} = entry;\n  const tag = entry.tag as ChangeTag;\n  switch (tag) {\n    case 'begin':\n      return [\n        watermark,\n        tag,\n        `[\"begin\",${change},{\"commitWatermark\":\"${watermark}\"}]`,\n      ];\n    case 'commit':\n      return [\n        watermark,\n        tag,\n        `[\"commit\",${change},{\"watermark\":\"${watermark}\"}]`,\n      ];\n    case 'rollback':\n      return [watermark, tag, `[\"rollback\",${change}]`];\n    default:\n      return [watermark, tag, `[\"data\",${change}]`];\n  }\n}\n"],"mappings":";;;AAWA,SAAgB,0BAA0B,MAAgC;CACxE,OAAO,WAAW,UAAU,IAAI;AAClC;;;;;;;AAQA,SAAgB,uBACd,mBACA,KACQ;CACR,QAAQ,KAAR;EACE,KAAK;EACL,KAAK,UAIH,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;EACF,SAEE,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;CACJ;AACF;;AAGA,SAAgB,6BACd,OACmB;CACnB,MAAM,EAAC,WAAW,WAAU;CAC5B,MAAM,MAAM,MAAM;CAClB,QAAQ,KAAR;EACE,KAAK,SACH,OAAO;GACL;GACA;GACA,YAAY,OAAO,uBAAuB,UAAU;EACtD;EACF,KAAK,UACH,OAAO;GACL;GACA;GACA,aAAa,OAAO,iBAAiB,UAAU;EACjD;EACF,KAAK,YACH,OAAO;GAAC;GAAW;GAAK,eAAe,OAAO;EAAE;EAClD,SACE,OAAO;GAAC;GAAW;GAAK,WAAW,OAAO;EAAE;CAChD;AACF"}
+{"version":3,"file":"change-log-codec.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-codec.ts"],"sourcesContent":["import {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer.ts';\n\nexport type ChangeLogEntry = {\n  watermark: string;\n  tag: string;\n  change: string;\n};\n\nexport type SerializedChangeStreamData = {\n  /** The canonical downstream representation forwarded to subscribers. */\n  readonly json: string;\n  /** The already-serialized change tuple element stored in change logs. */\n  readonly change: string;\n};\n\n/**\n * Serializes a data-plane message and its stored change payload in one pass.\n *\n * Change logs retain only the second tuple element for backwards\n * compatibility. Serializing that element first avoids stringifying the full\n * message and then copying the payload back out of the resulting string on the\n * replication hot path.\n */\nexport function serializeChangeStreamDataWithChange(\n  data: ChangeStreamData,\n): SerializedChangeStreamData {\n  const change = BigIntJSON.stringify(data[1]);\n  const type = data[0];\n  switch (type) {\n    case 'begin':\n    case 'commit':\n      return {\n        json: `[${JSON.stringify(type)},${change},${BigIntJSON.stringify(data[2])}]`,\n        change,\n      };\n    case 'data':\n    case 'rollback':\n      return {json: `[${JSON.stringify(type)},${change}]`, change};\n  }\n}\n\n/** Serializes a data-plane message to the canonical downstream JSON form. */\nexport function serializeChangeStreamData(data: ChangeStreamData): string {\n  return serializeChangeStreamDataWithChange(data).json;\n}\n\n/**\n * Extracts the second tuple element for callers that only have the canonical\n * downstream JSON. The live write path uses\n * {@link serializeChangeStreamDataWithChange} and does not scan or copy the\n * completed message.\n */\nexport function extractChangeSubstring(\n  streamMessageJSON: string,\n  tag: ChangeTag | undefined,\n): string {\n  switch (tag) {\n    case 'begin':\n    case 'commit':\n      // e.g.\n      // [\"begin\",<message-json>,{\"commitWatermark\":\"92fj2d0s\"}]\n      // [\"commit\",<message-json>,{\"watermark\":\"92fj2d0s\"}]\n      return streamMessageJSON.substring(\n        streamMessageJSON.indexOf(',') + 1,\n        streamMessageJSON.lastIndexOf(','),\n      );\n    default:\n      // [\"data\",<message-json>]\n      return streamMessageJSON.substring(\n        streamMessageJSON.indexOf(',') + 1,\n        streamMessageJSON.lastIndexOf(']'),\n      );\n  }\n}\n\n/** Reconstructs the canonical downstream JSON around a stored change. */\nexport function reconstructWatermarkedChange(\n  entry: ChangeLogEntry,\n): WatermarkedChange {\n  const {watermark, change} = entry;\n  const tag = entry.tag as ChangeTag;\n  switch (tag) {\n    case 'begin':\n      return [\n        watermark,\n        tag,\n        `[\"begin\",${change},{\"commitWatermark\":\"${watermark}\"}]`,\n      ];\n    case 'commit':\n      return [\n        watermark,\n        tag,\n        `[\"commit\",${change},{\"watermark\":\"${watermark}\"}]`,\n      ];\n    case 'rollback':\n      return [watermark, tag, `[\"rollback\",${change}]`];\n    default:\n      return [watermark, tag, `[\"data\",${change}]`];\n  }\n}\n"],"mappings":";;;;;;;;;;AAyBA,SAAgB,oCACd,MAC4B;CAC5B,MAAM,SAAS,WAAW,UAAU,KAAK,EAAE;CAC3C,MAAM,OAAO,KAAK;CAClB,QAAQ,MAAR;EACE,KAAK;EACL,KAAK,UACH,OAAO;GACL,MAAM,IAAI,KAAK,UAAU,IAAI,EAAE,GAAG,OAAO,GAAG,WAAW,UAAU,KAAK,EAAE,EAAE;GAC1E;EACF;EACF,KAAK;EACL,KAAK,YACH,OAAO;GAAC,MAAM,IAAI,KAAK,UAAU,IAAI,EAAE,GAAG,OAAO;GAAI;EAAM;CAC/D;AACF;;;;;;;AAaA,SAAgB,uBACd,mBACA,KACQ;CACR,QAAQ,KAAR;EACE,KAAK;EACL,KAAK,UAIH,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;EACF,SAEE,OAAO,kBAAkB,UACvB,kBAAkB,QAAQ,GAAG,IAAI,GACjC,kBAAkB,YAAY,GAAG,CACnC;CACJ;AACF;;AAGA,SAAgB,6BACd,OACmB;CACnB,MAAM,EAAC,WAAW,WAAU;CAC5B,MAAM,MAAM,MAAM;CAClB,QAAQ,KAAR;EACE,KAAK,SACH,OAAO;GACL;GACA;GACA,YAAY,OAAO,uBAAuB,UAAU;EACtD;EACF,KAAK,UACH,OAAO;GACL;GACA;GACA,aAAa,OAAO,iBAAiB,UAAU;EACjD;EACF,KAAK,YACH,OAAO;GAAC;GAAW;GAAK,eAAe,OAAO;EAAE;EAClD,SACE,OAAO;GAAC;GAAW;GAAK,WAAW,OAAO;EAAE;CAChD;AACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts

@@ -1,13 +1,17 @@
 import type { IncomingMessage } from 'node:http';
 import type { LogContext } from '@rocicorp/logger';
+import type { NormalizedZeroConfig } from '../../config/normalize.ts';
 import { type Worker } from '../../types/processes.ts';
 import { type ShardID } from '../../types/shards.ts';
-import { type Source } from '../../types/streams.ts';
+import { type Sized, type Source } from '../../types/streams.ts';
+import { Subscription } from '../../types/subscription.ts';
 import { HttpService, type Options as HttpOptions } from '../http-service.ts';
-import { type ChangeStreamer, type ChangeStreamerService, type SizedDownstream, type SubscriberContext } from './change-streamer.ts';
-import { type SnapshotMessage } from './snapshot.ts';
+import { type ChangeStreamer, type ChangeStreamerService, type SubscriberContext } from './change-streamer.ts';
+import { type SnapshotStatus, type SubscribeDownstream, type SubscribeUpstream } from './subscribe.ts';
 type Options = HttpOptions & {
     startupDelayMs: number;
+    config?: Pick<NormalizedZeroConfig, 'adminPassword' | 'operatorPassword'> | undefined;
+    getProfileWorker?: (() => Promise<Worker>) | undefined;
 };
 export declare class ChangeStreamerHttpServer extends HttpService {
     #private;
@@ -16,11 +20,49 @@ export declare class ChangeStreamerHttpServer extends HttpService {
     protected _onStart(): void;
     protected _onStop(): Promise<void>;
 }
-export declare class ChangeStreamerHttpClient implements ChangeStreamer {
+export interface SnapshotReserver {
+    reserveSnapshot(taskID: string): Promise<{
+        reserved: SnapshotStatus;
+        followup: ReservationFollowup;
+    }>;
+}
+export interface SubscriptionStarter {
+    subscribe(ctx: SubscriberContext): Promise<Source<Sized<SubscribeDownstream>>>;
+}
+export interface ReservationFollowup extends SubscriptionStarter {
+    /**
+     * A signal to indicate that the followup is no longer valid (i.e. the
+     * connection to the reserving task has been severed).
+     */
+    readonly signal: AbortSignal;
+    /**
+     * Abandons the reservation without subscribing, closing the connection that
+     * was held open to pin the change-log floor. Used when the caller cannot
+     * proceed to `subscribe()` (e.g. process shutdown while a restore is in
+     * progress) and must not leave the reservation's connection open forever.
+     */
+    cancel(reason?: Error): void;
+}
+export declare class ChangeStreamerHttpClient implements ChangeStreamer, SnapshotReserver, SubscriptionStarter {
     #private;
     constructor(lc: LogContext, shardID: ShardID, changeDB: string, changeStreamerURI: string | undefined);
-    reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>>;
-    subscribe(ctx: SubscriberContext): Promise<Source<SizedDownstream>>;
+    reserveSnapshot(taskID: string): Promise<{
+        reserved: SnapshotStatus;
+        followup: ReservationFollowup;
+    }>;
+    subscribe(ctx: SubscriberContext): Promise<Source<Sized<SubscribeDownstream>>>;
+    /**
+     * Opens a merged v7 `/subscribe` connection (see subscribe.ts). Returns the
+     * inbound change/reservation stream and an `outbound` sink into which the
+     * caller pushes the `reserve-snapshot` / `start-subscription` control
+     * messages that drive the connection's phases. The single connection spans
+     * the reservation, backup restore, and subscription, so the reserving
+     * replication-manager is the one that serves the subscription.
+     */
+    connect(taskID: string): Promise<{
+        instream: Source<Sized<SubscribeDownstream>>;
+        outbound: Subscription<SubscribeUpstream>;
+    }>;
 }
 type RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;
 export declare function getSubscriberContext(req: RequestHeaders): SubscriberContext;

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAKL,KAAK,MAAM,EACZ,MAAM,wBAAwB,CAAC;AAIhC,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,IAAI,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAwB,KAAK,eAAe,EAAC,MAAM,eAAe,CAAC;AAW1E,KAAK,OAAO,GAAG,WAAW,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;IAK5C,YACE,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,qBAAqB,EAuBtC;IAyED,UAAmB,QAAQ,IAAI,IAAI,CA2BlC;IAED,UAAyB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAIhD;CACF;AAED,qBAAa,wBAAyB,YAAW,cAAc;;IAM7D,YACE,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS,EAWtC;IAmBK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAOtE;IAEK,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAOxE;CACF;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAkB3E"}
+{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAGpE,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAKL,KAAK,KAAK,EACV,KAAK,MAAM,EACZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAIzD,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,IAAI,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAEL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAIL,KAAK,cAAc,EAEnB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACvB,MAAM,gBAAgB,CAAC;AAexB,KAAK,OAAO,GAAG,WAAW,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EACH,IAAI,CAAC,oBAAoB,EAAE,eAAe,GAAG,kBAAkB,CAAC,GAChE,SAAS,CAAC;IACd,gBAAgB,CAAC,EAAE,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;CACxD,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;IAK5C,YACE,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,qBAAqB,EAuCtC;IAiLD,UAAmB,QAAQ,IAAI,IAAI,CA2BlC;IAED,UAAyB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAIhD;CACF;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QACvC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,mBAAmB,CAAC;KAC/B,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,CACP,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,mBAAoB,SAAQ,mBAAmB;IAC9D;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAE7B;;;;;OAKG;IACH,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;CAC9B;AAED,qBAAa,wBACX,YAAW,cAAc,EAAE,gBAAgB,EAAE,mBAAmB;;IAOhE,YACE,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS,EAWtC;IAmBK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAC7C,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,mBAAmB,CAAC;KAC/B,CAAC,CAwBD;IAEK,SAAS,CACb,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAI7C;IAED;;;;;;;OAOG;IAEG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QACrC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAC7C,QAAQ,EAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;KAC3C,CAAC,CAgBD;CACF;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAiB3E"}