@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.2

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js

@@ -1,15 +1,38 @@
 import { forkChildWorker } from "../../types/processes.js";
-import { BACKUP_WATERMARK_READER_URL } from "../../server/worker-urls.js";
+import { BACKUP_WATERMARK_POLLER_URL } from "../../server/worker-urls.js";
+import { getVfsEnv } from "../litestream/vfs-watermark-poller.js";
+import { Subscription } from "../../types/subscription.js";
 import { getLastBackupTime } from "../litestream/commands.js";
-import { VfsBackupWatermarkWorkerSource } from "../litestream/vfs-watermark-worker-source.js";
-import { Litestream3BackupMonitor } from "./litestream3-backup-monitor.js";
-import { VfsBackupMonitor } from "./vfs-backup-monitor.js";
+import { BackupMonitor } from "./backup-monitor.js";
+import { Litestream3PrometheusPoller } from "./litestream3-prometheus-poller.js";
+import { ReplicaPoller } from "./replica-poller.js";
 //#region ../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts
-function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, initialCleanupDelayMs, verifyBackupState, vfsBackupWatermarkSource, env }) {
-	const { backupURL, port: metricsPort } = config.litestream;
-	if (!backupURL) return null;
-	if (config.litestream.backupUsingV5) return new VfsBackupMonitor(lc, backupURL, changeStreamer, initialCleanupDelayMs, config.litestream.vfsProbeIntervalMs, vfsBackupWatermarkSource ?? new VfsBackupWatermarkWorkerSource(lc, () => forkChildWorker(BACKUP_WATERMARK_READER_URL, env ?? process.env), config.litestream.vfsProbeTimeoutMs));
-	return new Litestream3BackupMonitor(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, changeStreamer, initialCleanupDelayMs, verifyBackupState ?? (() => getLastBackupTime(lc, config)));
+function createBackupCleanupMonitor({ lc, config, processes, replicaFile, changeStreamer, verifyBackupState, env = process.env }) {
+	const { litestream, replica } = config;
+	const { backupURL } = litestream;
+	let stream;
+	if (!backupURL) stream = new ReplicaPoller(lc, replicaFile).start();
+	else if (config.litestream.backupUsingV5) {
+		const { logLevel, endpoint, region, vfsLogFile: logFile, vfsProbeIntervalMs: remotePollIntervalMs } = litestream;
+		const sub = Subscription.create();
+		const vfsEnv = getVfsEnv({
+			backupURL,
+			endpoint,
+			region,
+			logLevel,
+			logFile,
+			remotePollIntervalMs
+		});
+		processes.addWorker(forkChildWorker(BACKUP_WATERMARK_POLLER_URL, {
+			...env,
+			...vfsEnv
+		}), "supporting", "backup-watermark-poller").onMessageType("backupWatermarkUpdate", (msg) => sub.push(msg));
+		stream = sub;
+	} else {
+		const { port: metricsPort } = litestream;
+		stream = new Litestream3PrometheusPoller(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, verifyBackupState ?? (() => getLastBackupTime(lc, litestream, replica.file))).start();
+	}
+	return new BackupMonitor(lc, stream, changeStreamer, replicaFile);
 }
 //#endregion
 export { createBackupCleanupMonitor };

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {BACKUP_WATERMARK_READER_URL} from '../../server/worker-urls.ts';\nimport {forkChildWorker} from '../../types/processes.ts';\nimport {getLastBackupTime} from '../litestream/commands.ts';\nimport {VfsBackupWatermarkWorkerSource} from '../litestream/vfs-watermark-worker-source.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {\n  Litestream3BackupMonitor,\n  type BackupStateVerifier,\n} from './litestream3-backup-monitor.ts';\nimport {\n  VfsBackupMonitor,\n  type VfsBackupWatermarkSource,\n} from './vfs-backup-monitor.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  initialCleanupDelayMs: number;\n  verifyBackupState?: BackupStateVerifier | undefined;\n  vfsBackupWatermarkSource?: VfsBackupWatermarkSource | undefined;\n  env?: NodeJS.ProcessEnv | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  replicaFile,\n  changeStreamer,\n  initialCleanupDelayMs,\n  verifyBackupState,\n  vfsBackupWatermarkSource,\n  env,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor | null {\n  const {backupURL, port: metricsPort} = config.litestream;\n  if (!backupURL) {\n    return null;\n  }\n\n  if (config.litestream.backupUsingV5) {\n    return new VfsBackupMonitor(\n      lc,\n      backupURL,\n      changeStreamer,\n      initialCleanupDelayMs,\n      config.litestream.vfsProbeIntervalMs,\n      vfsBackupWatermarkSource ??\n        new VfsBackupWatermarkWorkerSource(\n          lc,\n          () =>\n            forkChildWorker(BACKUP_WATERMARK_READER_URL, env ?? process.env),\n          config.litestream.vfsProbeTimeoutMs,\n        ),\n    );\n  }\n\n  return new Litestream3BackupMonitor(\n    lc,\n    replicaFile,\n    backupURL,\n    `http://localhost:${metricsPort}/metrics`,\n    changeStreamer,\n    initialCleanupDelayMs,\n    verifyBackupState ?? (() => getLastBackupTime(lc, config)),\n  );\n}\n"],"mappings":";;;;;;;AA4BA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,uBACA,mBACA,0BACA,OAC2D;CAC3D,MAAM,EAAC,WAAW,MAAM,gBAAe,OAAO;CAC9C,IAAI,CAAC,WACH,OAAO;CAGT,IAAI,OAAO,WAAW,eACpB,OAAO,IAAI,iBACT,IACA,WACA,gBACA,uBACA,OAAO,WAAW,oBAClB,4BACE,IAAI,+BACF,UAEE,gBAAgB,6BAA6B,OAAO,QAAQ,GAAG,GACjE,OAAO,WAAW,iBACpB,CACJ;CAGF,OAAO,IAAI,yBACT,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,gBACA,uBACA,4BAA4B,kBAAkB,IAAI,MAAM,EAC1D;AACF"}
+{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {BACKUP_WATERMARK_POLLER_URL} from '../../server/worker-urls.ts';\nimport {forkChildWorker} from '../../types/processes.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {ProcessManager} from '../life-cycle.ts';\nimport {getLastBackupTime} from '../litestream/commands.ts';\nimport {\n  type BackupWatermarkUpdate,\n  getVfsEnv,\n} from '../litestream/vfs-watermark-poller.ts';\nimport {type BackedUpWatermark, BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {\n  type BackupStateVerifier,\n  Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  processes: ProcessManager;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  verifyBackupState?: BackupStateVerifier | undefined;\n  env?: NodeJS.ProcessEnv | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  processes,\n  replicaFile,\n  changeStreamer,\n  verifyBackupState,\n  env = process.env,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n  const {litestream, replica} = config;\n  const {backupURL} = litestream;\n\n  let stream: Source<BackedUpWatermark>;\n\n  if (!backupURL) {\n    stream = new ReplicaPoller(lc, replicaFile).start();\n  } else if (config.litestream.backupUsingV5) {\n    const {\n      logLevel,\n      endpoint,\n      region,\n      vfsLogFile: logFile,\n      vfsProbeIntervalMs: remotePollIntervalMs,\n    } = litestream;\n    const sub = Subscription.create<BackedUpWatermark>();\n    const vfsEnv = getVfsEnv({\n      backupURL,\n      endpoint,\n      region,\n      logLevel,\n      logFile,\n      remotePollIntervalMs,\n    });\n    processes\n      .addWorker(\n        forkChildWorker(BACKUP_WATERMARK_POLLER_URL, {...env, ...vfsEnv}),\n        'supporting',\n        'backup-watermark-poller',\n      )\n      .onMessageType<BackupWatermarkUpdate>('backupWatermarkUpdate', msg =>\n        sub.push(msg),\n      );\n    stream = sub;\n  } else {\n    const {port: metricsPort} = litestream;\n    stream = new Litestream3PrometheusPoller(\n      lc,\n      replicaFile,\n      backupURL,\n      `http://localhost:${metricsPort}/metrics`,\n      verifyBackupState ??\n        (() => getLastBackupTime(lc, litestream, replica.file)),\n    ).start();\n  }\n\n  return new BackupMonitor(lc, stream, changeStreamer, replicaFile);\n}\n"],"mappings":";;;;;;;;;AA8BA,SAAgB,2BAA2B,EACzC,IACA,QACA,WACA,aACA,gBACA,mBACA,MAAM,QAAQ,OACsC;CACpD,MAAM,EAAC,YAAY,YAAW;CAC9B,MAAM,EAAC,cAAa;CAEpB,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW,EAAE,MAAM;MAC7C,IAAI,OAAO,WAAW,eAAe;EAC1C,MAAM,EACJ,UACA,UACA,QACA,YAAY,SACZ,oBAAoB,yBAClB;EACJ,MAAM,MAAM,aAAa,OAA0B;EACnD,MAAM,SAAS,UAAU;GACvB;GACA;GACA;GACA;GACA;GACA;EACF,CAAC;EACD,UACG,UACC,gBAAgB,6BAA6B;GAAC,GAAG;GAAK,GAAG;EAAM,CAAC,GAChE,cACA,yBACF,EACC,cAAqC,0BAAyB,QAC7D,IAAI,KAAK,GAAG,CACd;EACF,SAAS;CACX,OAAO;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD,EAAE,MAAM;CACV;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,gBAAgB,WAAW;AAClE"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts

@@ -1,8 +1,18 @@
-import type { Subscription } from '../../types/subscription.ts';
+import type { LogContext } from '@rocicorp/logger';
+import type { Source } from '../../types/streams.ts';
 import type { SingletonService } from '../service.ts';
-import type { SnapshotMessage } from './snapshot.ts';
-export interface BackupMonitor extends SingletonService {
-    startSnapshotReservation(taskID: string): Subscription<SnapshotMessage>;
-    endReservation(taskID: string, updateCleanupDelay?: boolean | undefined): void;
+import type { ChangeStreamerService } from './change-streamer.ts';
+export type BackedUpWatermark = {
+    watermark: string;
+    writeTimeMs?: number | undefined;
+    backupTimeMs: number;
+};
+export declare class BackupMonitor implements SingletonService {
+    #private;
+    readonly id = "backup-monitor";
+    constructor(lc: LogContext, watermarks: Source<BackedUpWatermark>, changeStreamer: ChangeStreamerService, replicaFile: string);
+    firstBackupReceived(): Promise<void>;
+    run(): Promise<void>;
+    stop(): Promise<void>;
 }
 //# sourceMappingURL=backup-monitor.d.ts.map

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,MAAM,WAAW,aAAc,SAAQ,gBAAgB;IACrD,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC,CAAC;IACxE,cAAc,CACZ,MAAM,EAAE,MAAM,EACd,kBAAkB,CAAC,EAAE,OAAO,GAAG,SAAS,GACvC,IAAI,CAAC;CACT"}
+{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,aAAc,YAAW,gBAAgB;;IACpD,QAAQ,CAAC,EAAE,oBAAoB;gBAU7B,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,EACrC,cAAc,EAAE,qBAAqB,EACrC,WAAW,EAAE,MAAM;IAQrB,mBAAmB;IAIb,GAAG;IAeT,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAkCtB"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js

@@ -0,0 +1,66 @@
+import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { getOrCreateGauge } from "../../observability/metrics.js";
+import { resolver } from "@rocicorp/resolver";
+//#region ../zero-cache/src/services/change-streamer/backup-monitor.ts
+var BackupMonitor = class {
+	id = "backup-monitor";
+	#lc;
+	#watermarks;
+	#changeStreamer;
+	#replicaFile;
+	#firstBackupReceived = resolver();
+	#latestBackup;
+	constructor(lc, watermarks, changeStreamer, replicaFile) {
+		this.#lc = lc;
+		this.#watermarks = watermarks;
+		this.#changeStreamer = changeStreamer;
+		this.#replicaFile = replicaFile;
+	}
+	firstBackupReceived() {
+		return this.#firstBackupReceived.promise;
+	}
+	async run() {
+		this.#lc.info?.("starting backup monitor");
+		this.#initBackupLagMetric();
+		for await (const backedUp of this.#watermarks) {
+			if (!this.#latestBackup) {
+				this.#lc.info?.(`received first backup up watermark`, { backedUp });
+				this.#firstBackupReceived.resolve();
+			}
+			this.#latestBackup = backedUp;
+			this.#changeStreamer.trackBackupWatermark(backedUp.watermark);
+		}
+		this.#lc.info?.("watermark stream closed. BackupMonitor stopped.");
+	}
+	stop() {
+		this.#watermarks.cancel();
+		return promiseVoid;
+	}
+	#initBackupLagMetric() {
+		getOrCreateGauge("replica", "backup_lag", {
+			description: "Latency from when a change is written to the replica to when it is backed up to litestream. It is expected to create a saw pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.",
+			unit: "millisecond"
+		}).addCallback((o) => {
+			const latestBackup = this.#latestBackup;
+			if (!latestBackup) {
+				this.#lc.warn?.(`no backed up watermarks. unable to report replica.backup_lag`);
+				return;
+			}
+			const db = new Database(this.#lc, this.#replicaFile, { readonly: true });
+			try {
+				const { writeTimeMs } = db.prepare(`SELECT writeTimeMs FROM "_zero.replicationState"`).get();
+				const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);
+				o.observe(backupLag);
+			} catch (e) {
+				this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);
+			} finally {
+				db.close();
+			}
+		});
+	}
+};
+//#endregion
+export { BackupMonitor };
+
+//# sourceMappingURL=backup-monitor.js.map

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#latestBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debuggin; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n\n    for await (const backedUp of this.#watermarks) {\n      if (!this.#latestBackup) {\n        this.#lc.info?.(`received first backup up watermark`, {backedUp});\n        this.#firstBackupReceived.resolve();\n      }\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAKC,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAKJ,IAAI,OAAO,yBAAyB;EACzC,KAAKK,qBAAqB;EAE1B,WAAW,MAAM,YAAY,KAAKJ,aAAa;GAC7C,IAAI,CAAC,KAAKK,eAAe;IACvB,KAAKN,IAAI,OAAO,sCAAsC,EAAC,SAAQ,CAAC;IAChE,KAAKI,qBAAqB,QAAQ;GACpC;GACA,KAAKE,gBAAgB;GACrB,KAAKJ,gBAAgB,qBAAqB,SAAS,SAAS;EAC9D;EACA,KAAKF,IAAI,OAAO,iDAAiD;CACnE;CAEA,OAAsB;EACpB,KAAKC,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,eAAe,KAAKK;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAKN,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKH,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts

@@ -1,17 +1,76 @@
 import type { LogContext } from '@rocicorp/logger';
-import type { WatermarkedChange } from './change-streamer-service.ts';
+import type { WatermarkedChange } from './change-streamer.ts';
 import type { Subscriber } from './subscriber.ts';
 export type BroadcastReleaseMode = 'all-subscribers' | 'consensus-timeout';
+/**
+ * Enables event-driven early release of a {@link Broadcast}. When provided and a
+ * majority has acked, the broadcast schedules its own consensus-timeout release
+ * proportional to how long the majority took to ack, instead of relying on the
+ * Forwarder's periodic progress tick.
+ */
+export type EarlyReleaseOptions = {
+    /** Time to wait, proportional to majority completion, before releasing. */
+    consensusTimeoutProportion: number;
+    /** Injectable timer, for deterministic testing. */
+    setTimeoutFn?: typeof setTimeout;
+    /** Injectable timer cancellation, for deterministic testing. */
+    clearTimeoutFn?: typeof clearTimeout;
+};
 /**
  * Initiates and tracks the progress of a change broadcasted to
- * a set of subscribers.
+ * a set of subscribers, implementing **consensus-based timeouts**
+ * to effect short-term graceful degradation when a minority of
+ * subscribers are behind.
+ *
+ * ### Background
+ *
+ * The purpose of flow control is to pull upstream replication changes
+ * no faster than the rate they are processed by downstream subscribers
+ * in the steady state. In the change-streamer, this is done by occasionally
+ * waiting for ACKs from subscribers before continuing; without doing so,
+ * I/O buffers fill up and cause the system to spend most of its time in GC.
+ *
+ * However, the naive algorithm of always waiting for all subscribers (e.g.
+ * `Promise.all()`) can behave poorly in scenarios where subscribers
+ * are imbalanced:
+ * * New subscribers may have a backlog of changes to catch up with.
+ *   Having all subscribers wait for the new subscriber to catch up results
+ *   in delaying the entire application.
+ * * Broken TCP connections similarly require all subscribers to wait until
+ *   connection liveness checks kick in and disconnect the subscriber.
  *
- * Creating a `Broadcast` automatically initiates the send.
+ * A simplistic approach is to add a limit to the amount of time waiting for
+ * subscribers, i.e. an ack timeout. However, deciding what this timeout
+ * should be is non-trivial because of the heterogeneous nature of changes;
+ * while most changes operate on single rows and are relatively predictable
+ * in terms of running time, some changes are table-wide operations and can
+ * legitimately take an arbitrary amount of time. In such scenarios, a
+ * timeout that is too short can stop progress on replication altogether.
  *
- * By default, {@link Broadcast.done} resolves when all subscribers
- * have acked the change. However, {@link Broadcast.checkProgress()}
- * can be called to resolve the broadcast earlier based on the flow
- * control policy.
+ * ### Consensus-based Timeout Algorithm
+ *
+ * To address these shortcomings, a "consensus-based timeout" algorithm is
+ * used:
+ * * Wait for more than half of the subscribers to finish. (In
+ *   case of a single node, or the case of one replication-manager
+ *   and one view-syncer, this reduces to waiting for all subscribers.)
+ * * Once more than half of the subscribers have finished, proceed after
+ *   the configured timeout, proportional to the time it took for the
+ *   majority of subscribers to ack, even if not all subscribers have
+ *   finished.
+ *
+ * In other words, the subscribers themselves are used to determine the
+ * timeout of each batch of changes; the majority determines this when
+ * they complete, upon which a timeout is logically started.
+ *
+ * In the common case, the remaining subscribers finish soon afterward and
+ * the timeout never elapses. However, in pathological cases where a minority
+ * of subscribers have a disproportionate amount of load, some will still
+ * be processing (or otherwise unresponsive). These subscribers are given
+ * a bounded amount of time to catch up at each flushed batch, up to the
+ * timeout interval. This guarantees eventual catchup because the
+ * subscribers with a backlog of changes necessarily have a higher
+ * processing rate than the subscribers that finished (and are made to wait).
  */
 export declare class Broadcast {
     #private;
@@ -24,79 +83,10 @@ export declare class Broadcast {
      * Broadcasts the `change` to the `subscribers` and tracks their
      * completion.
      */
-    constructor(subscribers: Iterable<Subscriber>, change: WatermarkedChange);
+    constructor(lc: LogContext, subscribers: Iterable<Subscriber>, change: WatermarkedChange, earlyRelease?: EarlyReleaseOptions);
     get isDone(): boolean;
     get done(): Promise<void>;
     get releaseMode(): BroadcastReleaseMode;
-    /**
-     * Checks for pathological situations in which flow should be reenabled
-     * before all subscribers have acked.
-     *
-     * ### Background
-     *
-     * The purpose of flow control is to pull upstream replication changes
-     * no faster than the rate as they are processed by downstream subscribers
-     * in the steady state. In the change-streamer, this is done by occasionally
-     * waiting for ACKs from subscribers before continuing; without doing so,
-     * I/O buffers fill up and cause the system to spend most of its time in GC.
-     *
-     * However, the naive algorithm of always waiting for all subscribers (e.g.
-     * `Promise.all()`) can behave poorly in scenarios where subscribers
-     * are imbalanced:
-     * * New subscribers may have a backlog of changes to catch up with.
-     *   Having all subscribers wait for the new subscriber to catch up results
-     *   in delaying the entire application.
-     * * Broken TCP connections similarly require all subscribers to wait until
-     *   connection liveness checks kick in and disconnect the subscriber.
-     *
-     * A simplistic approach is to add a limit to the amount of time waiting for
-     * subscribers, i.e. an ack timeout. However, deciding what this timeout
-     * should be is non-trivial because of the heterogeneous nature of changes;
-     * while most changes operate on single rows and are relatively predictable
-     * in terms of running time, some changes are table-wide operations and can
-     * legitimately take an arbitrary amount of time. In such scenarios, a
-     * timeout that is too short can stop progress on replication altogether.
-     *
-     * ### Consensus-based Timeout Algorithm
-     *
-     * To address these shortcomings, a "consensus-based timeout" algorithm is
-     * used:
-     * * Wait for more than half of the subscribers to finish. (In
-     *   case of a single node, or the case of one replication-manager
-     *   and one view-syncer, this reduces to waiting for all subscribers.)
-     * * Once more than half of the subscribers have finished, proceed after
-     *   a fixed timeout elapses (e.g. 1 second), even if not all subscribers
-     *   have finished.
-     *
-     * In other words, the subscribers themselves are used to determine the
-     * timeout of each batch of changes; the majority determines this when
-     * they complete, upon which a timeout is logically started.
-     *
-     * In the common case, the remaining subscribers finish soon afterward and
-     * the timeout never elapses. However, in pathological cases where a minority
-     * of subscribers have a disproportionate amount of load, some will still
-     * be processing (or otherwise unresponsive). These subscribers are given
-     * a bounded amount of time to catch up at each flushed batch, up to the
-     * timeout interval. This guarantees eventual catchup because the
-     * subscribers with a backlog of changes necessarily have a higher
-     * processing rate than the subscribers that finished (and are made to wait).
-     *
-     * ### Not implemented: Broken connection detection
-     *
-     * If a subscriber has not made progress for a certain interval, the
-     * algorithm could theoretically drop it preemptively, supplementing the
-     * existing websocket-level liveness checks.
-     *
-     * However, a more reliable approach would be to change the replicator
-     * to use non-blocking writes, and subsequently increase the frequency of
-     * connection-level liveness checks. The current synchronous replica writes
-     * can delay both ping responsiveness and change progress arbitrarily (e.g.
-     * a large index creation); an independently liveness check that is not
-     * delayed by synchronous writes on the subscriber would be a more failsafe
-     * solution.
-     *
-     * @returns `true` if the broadcast was already done or was marked done.
-     */
-    checkProgress(lc: LogContext, flowControlConsensusPaddingMs: number, now: number): boolean;
+    logSlowRequests(now: number): void;
 }
 //# sourceMappingURL=broadcast.d.ts.map

package/out/zero-cache/src/services/change-streamer/broadcast.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB;IAmB3B;;;OAGG;gBACS,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,iBAAiB;IAsCxE,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoEG;IACH,aAAa,CACX,EAAE,EAAE,UAAU,EACd,6BAA6B,EAAE,MAAM,EACrC,GAAG,EAAE,MAAM;CAkDd"}
+{"version":3,"file":"broadcast.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/broadcast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAE3E;;;;;GAKG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,2EAA2E;IAC3E,0BAA0B,EAAE,MAAM,CAAC;IACnC,mDAAmD;IACnD,YAAY,CAAC,EAAE,OAAO,UAAU,CAAC;IACjC,gEAAgE;IAChE,cAAc,CAAC,EAAE,OAAO,YAAY,CAAC;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,SAAS;;IACpB;;;OAGG;IACH,MAAM,CAAC,eAAe,CACpB,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB;IA0B3B;;;OAGG;gBAED,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,EACjC,MAAM,EAAE,iBAAiB,EACzB,YAAY,CAAC,EAAE,mBAAmB;IA4EpC,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAExB;IAED,IAAI,WAAW,IAAI,oBAAoB,CAEtC;IAED,eAAe,CAAC,GAAG,EAAE,MAAM;CA8B5B"}

package/out/zero-cache/src/services/change-streamer/broadcast.js

@@ -2,14 +2,59 @@ import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-streamer/broadcast.ts
 /**
 * Initiates and tracks the progress of a change broadcasted to
-* a set of subscribers.
+* a set of subscribers, implementing **consensus-based timeouts**
+* to effect short-term graceful degradation when a minority of
+* subscribers are behind.
 *
-* Creating a `Broadcast` automatically initiates the send.
+* ### Background
 *
-* By default, {@link Broadcast.done} resolves when all subscribers
-* have acked the change. However, {@link Broadcast.checkProgress()}
-* can be called to resolve the broadcast earlier based on the flow
-* control policy.
+* The purpose of flow control is to pull upstream replication changes
+* no faster than the rate they are processed by downstream subscribers
+* in the steady state. In the change-streamer, this is done by occasionally
+* waiting for ACKs from subscribers before continuing; without doing so,
+* I/O buffers fill up and cause the system to spend most of its time in GC.
+*
+* However, the naive algorithm of always waiting for all subscribers (e.g.
+* `Promise.all()`) can behave poorly in scenarios where subscribers
+* are imbalanced:
+* * New subscribers may have a backlog of changes to catch up with.
+*   Having all subscribers wait for the new subscriber to catch up results
+*   in delaying the entire application.
+* * Broken TCP connections similarly require all subscribers to wait until
+*   connection liveness checks kick in and disconnect the subscriber.
+*
+* A simplistic approach is to add a limit to the amount of time waiting for
+* subscribers, i.e. an ack timeout. However, deciding what this timeout
+* should be is non-trivial because of the heterogeneous nature of changes;
+* while most changes operate on single rows and are relatively predictable
+* in terms of running time, some changes are table-wide operations and can
+* legitimately take an arbitrary amount of time. In such scenarios, a
+* timeout that is too short can stop progress on replication altogether.
+*
+* ### Consensus-based Timeout Algorithm
+*
+* To address these shortcomings, a "consensus-based timeout" algorithm is
+* used:
+* * Wait for more than half of the subscribers to finish. (In
+*   case of a single node, or the case of one replication-manager
+*   and one view-syncer, this reduces to waiting for all subscribers.)
+* * Once more than half of the subscribers have finished, proceed after
+*   the configured timeout, proportional to the time it took for the
+*   majority of subscribers to ack, even if not all subscribers have
+*   finished.
+*
+* In other words, the subscribers themselves are used to determine the
+* timeout of each batch of changes; the majority determines this when
+* they complete, upon which a timeout is logically started.
+*
+* In the common case, the remaining subscribers finish soon afterward and
+* the timeout never elapses. However, in pathological cases where a minority
+* of subscribers have a disproportionate amount of load, some will still
+* be processing (or otherwise unresponsive). These subscribers are given
+* a bounded amount of time to catch up at each flushed batch, up to the
+* timeout interval. This guarantees eventual catchup because the
+* subscribers with a backlog of changes necessarily have a higher
+* processing rate than the subscribers that finished (and are made to wait).
 */
 var Broadcast = class {
 	/**
@@ -19,6 +64,7 @@ var Broadcast = class {
 	static withoutTracking(subscribers, change) {
 		for (const sub of subscribers) sub.send(change);
 	}
+	#lc;
 	#pending;
 	#completed;
 	#done = resolver();
@@ -27,16 +73,23 @@ var Broadcast = class {
 	#watermark;
 	#majority;
 	#start = performance.now();
-	#latestCompleted = Number.MAX_VALUE;
+	#earlyReleaseTimeoutProportion;
+	#setTimeout;
+	#clearTimeout;
+	#earlyReleaseTimer;
 	/**
 	* Broadcasts the `change` to the `subscribers` and tracks their
 	* completion.
 	*/
-	constructor(subscribers, change) {
+	constructor(lc, subscribers, change, earlyRelease) {
+		this.#lc = lc;
 		this.#pending = new Set(subscribers);
 		this.#completed = [];
 		this.#watermark = change[0];
 		this.#majority = Math.floor(this.#pending.size / 2) + 1;
+		this.#earlyReleaseTimeoutProportion = earlyRelease?.consensusTimeoutProportion;
+		this.#setTimeout = earlyRelease?.setTimeoutFn ?? setTimeout;
+		this.#clearTimeout = earlyRelease?.clearTimeoutFn ?? clearTimeout;
 		for (const sub of this.#pending) {
 			const changes = sub.numPending + 1;
 			sub.send(change).catch(() => {}).finally(() => this.#markCompleted(sub, changes));
@@ -44,19 +97,35 @@ var Broadcast = class {
 		if (this.#pending.size === 0) this.#setDone("all-subscribers");
 	}
 	#markCompleted(sub, changes) {
-		const elapsed = (this.#latestCompleted = performance.now()) - this.#start;
+		if (this.#isDone) return;
+		const elapsed = performance.now() - this.#start;
+		sub.trackResponseResult("on-time");
 		this.#completed.push({
 			sub,
 			changes,
 			elapsed
 		});
 		this.#pending.delete(sub);
-		if (this.#pending.size === 0) this.#setDone("all-subscribers");
+		if (this.#pending.size === 0) {
+			this.#setDone("all-subscribers");
+			return;
+		}
+		if (this.#earlyReleaseTimeoutProportion !== void 0 && this.#earlyReleaseTimer === void 0 && this.#completed.length === this.#majority) {
+			const timeoutMs = Math.ceil(elapsed * this.#earlyReleaseTimeoutProportion);
+			this.#earlyReleaseTimer = this.#setTimeout(() => this.#setDone("consensus-timeout"), timeoutMs);
+		}
 	}
 	#setDone(releaseMode) {
 		if (this.#isDone) return;
+		if (releaseMode === "consensus-timeout") {
+			const elapsed = performance.now() - this.#start;
+			this.#logWithState(`continuing with ${this.#pending.size} subscriber(s) still pending`, elapsed);
+		}
+		this.#pending.forEach((sub) => sub.trackResponseResult("timed-out"));
 		this.#isDone = true;
 		this.#releaseMode = releaseMode;
+		this.#clearTimeout(this.#earlyReleaseTimer);
+		this.#earlyReleaseTimer = void 0;
 		this.#done.resolve();
 	}
 	get isDone() {
@@ -68,92 +137,15 @@ var Broadcast = class {
 	get releaseMode() {
 		return this.#releaseMode ?? "all-subscribers";
 	}
-	/**
-	* Checks for pathological situations in which flow should be reenabled
-	* before all subscribers have acked.
-	*
-	* ### Background
-	*
-	* The purpose of flow control is to pull upstream replication changes
-	* no faster than the rate as they are processed by downstream subscribers
-	* in the steady state. In the change-streamer, this is done by occasionally
-	* waiting for ACKs from subscribers before continuing; without doing so,
-	* I/O buffers fill up and cause the system to spend most of its time in GC.
-	*
-	* However, the naive algorithm of always waiting for all subscribers (e.g.
-	* `Promise.all()`) can behave poorly in scenarios where subscribers
-	* are imbalanced:
-	* * New subscribers may have a backlog of changes to catch up with.
-	*   Having all subscribers wait for the new subscriber to catch up results
-	*   in delaying the entire application.
-	* * Broken TCP connections similarly require all subscribers to wait until
-	*   connection liveness checks kick in and disconnect the subscriber.
-	*
-	* A simplistic approach is to add a limit to the amount of time waiting for
-	* subscribers, i.e. an ack timeout. However, deciding what this timeout
-	* should be is non-trivial because of the heterogeneous nature of changes;
-	* while most changes operate on single rows and are relatively predictable
-	* in terms of running time, some changes are table-wide operations and can
-	* legitimately take an arbitrary amount of time. In such scenarios, a
-	* timeout that is too short can stop progress on replication altogether.
-	*
-	* ### Consensus-based Timeout Algorithm
-	*
-	* To address these shortcomings, a "consensus-based timeout" algorithm is
-	* used:
-	* * Wait for more than half of the subscribers to finish. (In
-	*   case of a single node, or the case of one replication-manager
-	*   and one view-syncer, this reduces to waiting for all subscribers.)
-	* * Once more than half of the subscribers have finished, proceed after
-	*   a fixed timeout elapses (e.g. 1 second), even if not all subscribers
-	*   have finished.
-	*
-	* In other words, the subscribers themselves are used to determine the
-	* timeout of each batch of changes; the majority determines this when
-	* they complete, upon which a timeout is logically started.
-	*
-	* In the common case, the remaining subscribers finish soon afterward and
-	* the timeout never elapses. However, in pathological cases where a minority
-	* of subscribers have a disproportionate amount of load, some will still
-	* be processing (or otherwise unresponsive). These subscribers are given
-	* a bounded amount of time to catch up at each flushed batch, up to the
-	* timeout interval. This guarantees eventual catchup because the
-	* subscribers with a backlog of changes necessarily have a higher
-	* processing rate than the subscribers that finished (and are made to wait).
-	*
-	* ### Not implemented: Broken connection detection
-	*
-	* If a subscriber has not made progress for a certain interval, the
-	* algorithm could theoretically drop it preemptively, supplementing the
-	* existing websocket-level liveness checks.
-	*
-	* However, a more reliable approach would be to change the replicator
-	* to use non-blocking writes, and subsequently increase the frequency of
-	* connection-level liveness checks. The current synchronous replica writes
-	* can delay both ping responsiveness and change progress arbitrarily (e.g.
-	* a large index creation); an independently liveness check that is not
-	* delayed by synchronous writes on the subscriber would be a more failsafe
-	* solution.
-	*
-	* @returns `true` if the broadcast was already done or was marked done.
-	*/
-	checkProgress(lc, flowControlConsensusPaddingMs, now) {
-		if (this.#isDone) return true;
-		if (this.#pending.size === 0) return true;
+	logSlowRequests(now) {
+		if (this.#isDone || this.#pending.size === 0) return;
 		const elapsed = now - this.#start;
 		if (this.#completed.length < this.#majority) {
-			if (elapsed >= 1e3) this.#logWithState(lc, `waiting for at least ${this.#majority} subscribers to finish`, elapsed);
-			return false;
-		}
-		if (now - this.#latestCompleted >= flowControlConsensusPaddingMs) {
-			this.#logWithState(lc, `continuing with ${this.#pending.size} subscriber(s) still pending`, elapsed);
-			this.#setDone("consensus-timeout");
-			return true;
+			if (elapsed >= 1e3) this.#logWithState(`waiting for at least ${this.#majority} subscribers to finish`, elapsed);
 		}
-		return false;
 	}
-	#logWithState(lc, msg, elapsed) {
-		lc.withContext("watermark", this.#watermark).info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {
+	#logWithState(msg, elapsed) {
+		this.#lc.withContext("watermark", this.#watermark).info?.(`${msg} (${elapsed.toFixed(3)} ms)`, {
 			completed: this.#completed.map((d) => ({
 				id: d.sub.id,
 				processed: d.changes,