@rocicorp/zero 1.12.0-canary.5 → 1.12.0-canary.6

package/out/zero-cache/src/services/change-streamer/backup-monitor.js

@@ -1,15 +1,23 @@
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
 import { Database } from "../../../../zqlite/src/db.js";
 import { getOrCreateGauge } from "../../observability/metrics.js";
+import { Subscription } from "../../types/subscription.js";
 import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-streamer/backup-monitor.ts
+/**
+* Relays the watermarks backed up by a {@link BackupPoller} to the
+* change-streamer, which consumes them (see {@link backups}) and requests
+* prompt backups (see {@link requestSync}) as needed.
+*/
 var BackupMonitor = class {
 	id = "backup-monitor";
 	#lc;
-	#watermarks;
-	#changeStreamer;
+	#poller;
 	#replicaFile;
 	#firstBackupReceived = resolver();
+	#backups = Subscription.create({ coalesce: (curr) => curr });
+	#watermarks;
+	#stopped = false;
 	#latestBackup;
 	#firstBackupResolved = false;
 	/**
@@ -19,36 +27,61 @@ var BackupMonitor = class {
 	* "any watermark" rather than hanging startup forever.
 	*/
 	#coverageTarget;
-	constructor(lc, watermarks, changeStreamer, replicaFile) {
+	constructor(lc, poller, replicaFile) {
 		this.#lc = lc;
-		this.#watermarks = watermarks;
-		this.#changeStreamer = changeStreamer;
+		this.#poller = poller;
 		this.#replicaFile = replicaFile;
 	}
 	firstBackupReceived() {
 		return this.#firstBackupReceived.promise;
 	}
+	/**
+	* The stream of backed-up watermarks, each later than the last, coalesced
+	* to the latest one that has not been consumed. It ends when the monitor
+	* stops.
+	*/
+	backups() {
+		return this.#backups;
+	}
+	/**
+	* Requests that the replica be backed up through `watermark` promptly, if
+	* the poller supports it (see {@link BackupPoller.requestSync}). A no-op
+	* unless the monitor is running.
+	*/
+	requestSync(watermark) {
+		if (!this.#watermarks || this.#stopped || this.#latestBackup && this.#latestBackup.watermark >= watermark) return;
+		if (this.#poller.requestSync) {
+			this.#lc.info?.(`requesting backup through ${watermark}`);
+			this.#poller.requestSync(watermark);
+		}
+	}
 	async run() {
+		if (this.#stopped) return;
 		this.#lc.info?.("starting backup monitor");
 		this.#initBackupLagMetric();
 		this.#coverageTarget = this.#readReplicaStateVersion();
-		for await (const backedUp of this.#watermarks) {
-			if (this.#latestBackup) {
-				if (backedUp.watermark < this.#latestBackup.watermark) {
-					this.#lc.warn?.(`ignoring earlier backup watermark ${backedUp.watermark}`, { backedUp });
-					continue;
-				}
-				if (backedUp.watermark === this.#latestBackup.watermark) {
-					this.#lc.debug?.(`ignoring redundant backup watermark ${backedUp.watermark}`, { backedUp });
-					continue;
+		this.#watermarks = this.#poller.start();
+		try {
+			for await (const backedUp of this.#watermarks) {
+				if (this.#latestBackup) {
+					if (backedUp.watermark < this.#latestBackup.watermark) {
+						this.#lc.warn?.(`ignoring earlier backup watermark ${backedUp.watermark}`, { backedUp });
+						continue;
+					}
+					if (backedUp.watermark === this.#latestBackup.watermark) {
+						this.#lc.debug?.(`ignoring redundant backup watermark ${backedUp.watermark}`, { backedUp });
+						continue;
+					}
 				}
+				this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, { backedUp });
+				this.#latestBackup = backedUp;
+				this.#backups.push(backedUp);
+				this.#checkFirstBackupCovers(backedUp);
 			}
-			this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, { backedUp });
-			this.#latestBackup = backedUp;
-			this.#changeStreamer.trackBackupWatermark(backedUp.watermark);
-			this.#checkFirstBackupCovers(backedUp);
+			this.#lc.info?.("watermark stream closed. BackupMonitor stopped.");
+		} finally {
+			this.#backups.cancel();
 		}
-		this.#lc.info?.("watermark stream closed. BackupMonitor stopped.");
 	}
 	/**
 	* Resolves {@link firstBackupReceived} once the backup actually covers the
@@ -90,7 +123,9 @@ var BackupMonitor = class {
 		}
 	}
 	stop() {
-		this.#watermarks.cancel();
+		this.#stopped = true;
+		this.#watermarks?.cancel();
+		this.#backups.cancel();
 		return promiseVoid;
 	}
 	#initBackupLagMetric() {

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debugging; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n  #firstBackupResolved = false;\n  /**\n   * The replica's `stateVersion` when the monitor started, i.e. the point the\n   * backup has to reach before it covers what this task is about to serve.\n   * `undefined` when the replica could not be read, which degrades the gate to\n   * \"any watermark\" rather than hanging startup forever.\n   */\n  #coverageTarget: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n    this.#coverageTarget = this.#readReplicaStateVersion();\n\n    for await (const backedUp of this.#watermarks) {\n      if (this.#latestBackup) {\n        if (backedUp.watermark < this.#latestBackup.watermark) {\n          this.#lc.warn?.(\n            `ignoring earlier backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n        if (backedUp.watermark === this.#latestBackup.watermark) {\n          this.#lc.debug?.(\n            `ignoring redundant backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n      }\n      this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, {\n        backedUp,\n      });\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n      this.#checkFirstBackupCovers(backedUp);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  /**\n   * Resolves {@link firstBackupReceived} once the backup actually covers the\n   * replica, rather than on the first watermark of any value.\n   *\n   * A new backup destination starts empty and is filled by re-uploading the\n   * local LTX chain, so the first watermarks read back out of it are real but\n   * far behind the replica. Releasing the readiness gate on one of those lets\n   * the task serve against a backup that does not yet cover it, which is what\n   * demotes a restoring view-syncer to Postgres catchup.\n   *\n   * The target is pinned at startup rather than compared against the replica's\n   * current position, which keeps the gate satisfiable under sustained writes.\n   */\n  #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n    if (this.#firstBackupResolved) {\n      return;\n    }\n    const target = this.#coverageTarget;\n    if (target !== undefined && backedUp.watermark < target) {\n      this.#lc.info?.(`backup does not yet cover the replica`, {\n        backedUp,\n        coverageTarget: target,\n      });\n      return;\n    }\n    this.#firstBackupResolved = true;\n    this.#firstBackupReceived.resolve();\n  }\n\n  #readReplicaStateVersion(): string | undefined {\n    let db;\n    try {\n      db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      const {stateVersion} = db\n        .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n        .get<{stateVersion: string}>();\n      return stateVersion;\n    } catch (e) {\n      // Without a target the gate degrades to its previous behavior. That is\n      // strictly better than blocking startup on a replica we cannot read.\n      this.#lc.warn?.(\n        `unable to read the replica's stateVersion; ` +\n          `the initial backup gate will accept the first watermark`,\n        e,\n      );\n      return undefined;\n    } finally {\n      db?.close();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAK,MAAM;EACX,KAAK,cAAc;EACnB,KAAK,kBAAkB;EACvB,KAAK,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAK,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAK,IAAI,OAAO,yBAAyB;EACzC,KAAK,qBAAqB;EAC1B,KAAK,kBAAkB,KAAK,yBAAyB;EAErD,WAAW,MAAM,YAAY,KAAK,aAAa;GAC7C,IAAI,KAAK,eAAe;IACtB,IAAI,SAAS,YAAY,KAAK,cAAc,WAAW;KACrD,KAAK,IAAI,OACP,qCAAqC,SAAS,aAC9C,EAAC,SAAQ,CACX;KACA;IACF;IACA,IAAI,SAAS,cAAc,KAAK,cAAc,WAAW;KACvD,KAAK,IAAI,QACP,uCAAuC,SAAS,aAChD,EAAC,SAAQ,CACX;KACA;IACF;GACF;GACA,KAAK,IAAI,OAAO,6BAA6B,SAAS,aAAa,EACjE,SACF,CAAC;GACD,KAAK,gBAAgB;GACrB,KAAK,gBAAgB,qBAAqB,SAAS,SAAS;GAC5D,KAAK,wBAAwB,QAAQ;EACvC;EACA,KAAK,IAAI,OAAO,iDAAiD;CACnE;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAK,sBACP;EAEF,MAAM,SAAS,KAAK;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAK,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAK,uBAAuB;EAC5B,KAAK,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,CAAC,CACpE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAK,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAK,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK;GAClB,MAAM,eAAe,KAAK;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAK,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,CAAC,CACnE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAK,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}
+{"version":3,"file":"backup-monitor.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {SingletonService} from '../service.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debugging; available on v5\n  backupTimeMs: number;\n};\n\n/**\n * Determines which watermarks of the replica are backed up, by whatever means\n * the backup (or its absence) allows.\n */\nexport interface BackupPoller {\n  /**\n   * Starts polling, returning the stream of backed-up watermarks. Must only\n   * be called once.\n   */\n  start(): Source<BackedUpWatermark>;\n\n  /**\n   * Requests that the replica be backed up through `watermark` promptly,\n   * rather than at the poller's regular cadence, once the replica has reached\n   * it. The resulting watermark is emitted on the stream as usual. Pollers\n   * that cannot initiate backups do not implement this.\n   */\n  requestSync?(watermark: string): void;\n}\n\n/**\n * The interface through which the change-streamer consumes backed-up\n * watermarks and requests prompt backups (see {@link BackupMonitor}).\n */\nexport type BackupSource = Pick<BackupMonitor, 'backups' | 'requestSync'>;\n\n/**\n * Relays the watermarks backed up by a {@link BackupPoller} to the\n * change-streamer, which consumes them (see {@link backups}) and requests\n * prompt backups (see {@link requestSync}) as needed.\n */\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #poller: BackupPoller;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n  // Backed-up watermarks only move forward, so the consumer only needs the\n  // latest one. Coalescing keeps at most one outstanding, so that the monitor\n  // neither waits on, nor queues for, the consumer.\n  readonly #backups = Subscription.create<BackedUpWatermark>({\n    coalesce: curr => curr,\n  });\n\n  #watermarks: Source<BackedUpWatermark> | undefined;\n  #stopped = false;\n  #latestBackup: BackedUpWatermark | undefined;\n  #firstBackupResolved = false;\n  /**\n   * The replica's `stateVersion` when the monitor started, i.e. the point the\n   * backup has to reach before it covers what this task is about to serve.\n   * `undefined` when the replica could not be read, which degrades the gate to\n   * \"any watermark\" rather than hanging startup forever.\n   */\n  #coverageTarget: string | undefined;\n\n  constructor(lc: LogContext, poller: BackupPoller, replicaFile: string) {\n    this.#lc = lc;\n    this.#poller = poller;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  /**\n   * The stream of backed-up watermarks, each later than the last, coalesced\n   * to the latest one that has not been consumed. It ends when the monitor\n   * stops.\n   */\n  backups(): Source<BackedUpWatermark> {\n    return this.#backups;\n  }\n\n  /**\n   * Requests that the replica be backed up through `watermark` promptly, if\n   * the poller supports it (see {@link BackupPoller.requestSync}). A no-op\n   * unless the monitor is running.\n   */\n  requestSync(watermark: string) {\n    if (\n      !this.#watermarks ||\n      this.#stopped ||\n      (this.#latestBackup && this.#latestBackup.watermark >= watermark)\n    ) {\n      return;\n    }\n    if (this.#poller.requestSync) {\n      this.#lc.info?.(`requesting backup through ${watermark}`);\n      this.#poller.requestSync(watermark);\n    }\n  }\n\n  async run() {\n    if (this.#stopped) {\n      return;\n    }\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n    this.#coverageTarget = this.#readReplicaStateVersion();\n\n    this.#watermarks = this.#poller.start();\n    try {\n      for await (const backedUp of this.#watermarks) {\n        if (this.#latestBackup) {\n          if (backedUp.watermark < this.#latestBackup.watermark) {\n            this.#lc.warn?.(\n              `ignoring earlier backup watermark ${backedUp.watermark}`,\n              {backedUp},\n            );\n            continue;\n          }\n          if (backedUp.watermark === this.#latestBackup.watermark) {\n            this.#lc.debug?.(\n              `ignoring redundant backup watermark ${backedUp.watermark}`,\n              {backedUp},\n            );\n            continue;\n          }\n        }\n        this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, {\n          backedUp,\n        });\n        this.#latestBackup = backedUp;\n        this.#backups.push(backedUp);\n        this.#checkFirstBackupCovers(backedUp);\n      }\n      this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n    } finally {\n      this.#backups.cancel();\n    }\n  }\n\n  /**\n   * Resolves {@link firstBackupReceived} once the backup actually covers the\n   * replica, rather than on the first watermark of any value.\n   *\n   * A new backup destination starts empty and is filled by re-uploading the\n   * local LTX chain, so the first watermarks read back out of it are real but\n   * far behind the replica. Releasing the readiness gate on one of those lets\n   * the task serve against a backup that does not yet cover it, which is what\n   * demotes a restoring view-syncer to Postgres catchup.\n   *\n   * The target is pinned at startup rather than compared against the replica's\n   * current position, which keeps the gate satisfiable under sustained writes.\n   */\n  #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n    if (this.#firstBackupResolved) {\n      return;\n    }\n    const target = this.#coverageTarget;\n    if (target !== undefined && backedUp.watermark < target) {\n      this.#lc.info?.(`backup does not yet cover the replica`, {\n        backedUp,\n        coverageTarget: target,\n      });\n      return;\n    }\n    this.#firstBackupResolved = true;\n    this.#firstBackupReceived.resolve();\n  }\n\n  #readReplicaStateVersion(): string | undefined {\n    let db;\n    try {\n      db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      const {stateVersion} = db\n        .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n        .get<{stateVersion: string}>();\n      return stateVersion;\n    } catch (e) {\n      // Without a target the gate degrades to its previous behavior. That is\n      // strictly better than blocking startup on a replica we cannot read.\n      this.#lc.warn?.(\n        `unable to read the replica's stateVersion; ` +\n          `the initial backup gate will accept the first watermark`,\n        e,\n      );\n      return undefined;\n    } finally {\n      db?.close();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#stopped = true;\n    this.#watermarks?.cancel();\n    this.#backups.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;AA8CA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA,uBAAgC,SAAS;CAIzC,WAAoB,aAAa,OAA0B,EACzD,WAAU,SAAQ,KACpB,CAAC;CAED;CACA,WAAW;CACX;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YAAY,IAAgB,QAAsB,aAAqB;EACrE,KAAK,MAAM;EACX,KAAK,UAAU;EACf,KAAK,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAK,qBAAqB;CACnC;;;;;;CAOA,UAAqC;EACnC,OAAO,KAAK;CACd;;;;;;CAOA,YAAY,WAAmB;EAC7B,IACE,CAAC,KAAK,eACN,KAAK,YACJ,KAAK,iBAAiB,KAAK,cAAc,aAAa,WAEvD;EAEF,IAAI,KAAK,QAAQ,aAAa;GAC5B,KAAK,IAAI,OAAO,6BAA6B,WAAW;GACxD,KAAK,QAAQ,YAAY,SAAS;EACpC;CACF;CAEA,MAAM,MAAM;EACV,IAAI,KAAK,UACP;EAEF,KAAK,IAAI,OAAO,yBAAyB;EACzC,KAAK,qBAAqB;EAC1B,KAAK,kBAAkB,KAAK,yBAAyB;EAErD,KAAK,cAAc,KAAK,QAAQ,MAAM;EACtC,IAAI;GACF,WAAW,MAAM,YAAY,KAAK,aAAa;IAC7C,IAAI,KAAK,eAAe;KACtB,IAAI,SAAS,YAAY,KAAK,cAAc,WAAW;MACrD,KAAK,IAAI,OACP,qCAAqC,SAAS,aAC9C,EAAC,SAAQ,CACX;MACA;KACF;KACA,IAAI,SAAS,cAAc,KAAK,cAAc,WAAW;MACvD,KAAK,IAAI,QACP,uCAAuC,SAAS,aAChD,EAAC,SAAQ,CACX;MACA;KACF;IACF;IACA,KAAK,IAAI,OAAO,6BAA6B,SAAS,aAAa,EACjE,SACF,CAAC;IACD,KAAK,gBAAgB;IACrB,KAAK,SAAS,KAAK,QAAQ;IAC3B,KAAK,wBAAwB,QAAQ;GACvC;GACA,KAAK,IAAI,OAAO,iDAAiD;EACnE,UAAU;GACR,KAAK,SAAS,OAAO;EACvB;CACF;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAK,sBACP;EAEF,MAAM,SAAS,KAAK;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAK,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAK,uBAAuB;EAC5B,KAAK,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,CAAC,CACpE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAK,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAK,WAAW;EAChB,KAAK,aAAa,OAAO;EACzB,KAAK,SAAS,OAAO;EACrB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK;GAClB,MAAM,eAAe,KAAK;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAK,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,CAAC,CACnE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAK,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts

@@ -5,6 +5,7 @@ import type { ChangeSource } from '../change-source/change-source.ts';
 import type { LitestreamVersion } from '../litestream/metrics.ts';
 import { type ReplicationStatusPublisher } from '../replicator/replication-status.ts';
 import type { SubscriptionState } from '../replicator/schema/replication-state.ts';
+import type { BackupSource } from './backup-monitor.ts';
 import { type ChangeStreamerService, type SubscriberContext } from './change-streamer.ts';
 import { type SQLiteChangeLogCleanupGuard } from './sqlite-change-log-catchup.ts';
 import { type SQLiteChangeLogCompareOptions } from './sqlite-change-log-comparator.ts';
@@ -125,6 +126,14 @@ export type TuningOptions = StorerOptions & {
     }) | undefined;
     /** Supplied only in `serve` mode. A zero percentage keeps every read on PG. */
     sqliteChangeLogServe?: SQLiteChangeLogServeOptions | undefined;
+    /**
+     * The source of backed-up watermarks (i.e. the `BackupMonitor`), which the
+     * change-streamer tracks from construction, and from which it requests
+     * prompt backups when the change source asks for them (e.g. for a replica
+     * fork). Its lifecycle (e.g. the monitor's run() and stop()) is managed by
+     * the caller. Absent, no backups are tracked.
+     */
+    backupMonitor?: BackupSource | undefined;
 };
 /**
  * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAajF,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EAEvB,MAAM,sBAAsB,CAAC;AAa9B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAGL,KAAK,oCAAoC,EAC1C,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAIL,KAAK,uBAAuB,EAC7B,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAEL,KAAK,4BAA4B,EAClC,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAKrB,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9D;;;;OAIG;IACH,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,8DAA8D;IAC9D,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,MAAM,uBAAuB,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;CACnE,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C;;;;;;;;;OASG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5D;;;;;;OAMG;IACH,+BAA+B,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrD;;;;OAIG;IACH,uCAAuC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7D,aAAa,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACjD;;;OAGG;IACH,qBAAqB,CAAC,EAClB,IAAI,CACF,4BAA4B,EAC5B,UAAU,GAAG,YAAY,GAAG,WAAW,CACxC,GACD,SAAS,CAAC;IACd;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oCAAoC,GAAG,SAAS,CAAC;IACxE;;;;;;OAMG;IACH,sBAAsB,CAAC,EACnB,CAAC,6BAA6B,GAAG;QAAC,WAAW,EAAE,MAAM,CAAA;KAAC,CAAC,GACvD,SAAS,CAAC;IACd,+EAA+E;IAC/E,oBAAoB,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;CAChE,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM;AACzB;;;;;;;;GAQG;AACH,QAAQ,EAAE,UAAU,GAAG,IAAI,EAC3B,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CAoDhC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAOjF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAOtD,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EAEvB,MAAM,sBAAsB,CAAC;AAa9B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAGL,KAAK,oCAAoC,EAC1C,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAIL,KAAK,uBAAuB,EAC7B,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAEL,KAAK,4BAA4B,EAClC,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAKrB,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9D;;;;OAIG;IACH,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,8DAA8D;IAC9D,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,MAAM,uBAAuB,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;CACnE,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C;;;;;;;;;OASG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5D;;;;;;OAMG;IACH,+BAA+B,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrD;;;;OAIG;IACH,uCAAuC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7D,aAAa,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACjD;;;OAGG;IACH,qBAAqB,CAAC,EAClB,IAAI,CACF,4BAA4B,EAC5B,UAAU,GAAG,YAAY,GAAG,WAAW,CACxC,GACD,SAAS,CAAC;IACd;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oCAAoC,GAAG,SAAS,CAAC;IACxE;;;;;;OAMG;IACH,sBAAsB,CAAC,EACnB,CAAC,6BAA6B,GAAG;QAAC,WAAW,EAAE,MAAM,CAAA;KAAC,CAAC,GACvD,SAAS,CAAC;IACd,+EAA+E;IAC/E,oBAAoB,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IAC/D;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;CAC1C,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM;AACzB;;;;;;;;GAQG;AACH,QAAQ,EAAE,UAAU,GAAG,IAAI,EAC3B,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CAoDhC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -202,6 +202,7 @@ var ChangeStreamerImpl = class {
 	#purgeScheduler;
 	#comparator;
 	#acker;
+	#backupMonitor;
 	#initializer;
 	#readRouter;
 	#autoReset;
@@ -216,7 +217,7 @@ var ChangeStreamerImpl = class {
 	#reservationDemotions = getOrCreateCounter("replication", "sqlite_change_log.reservation_demotions", "Snapshot reservations demoted from SQLite to PG because the change log could not cover the backup being restored. With PG retired these followers have no fallback, so this is the rate at which one would instead have to wait for a later backup.");
 	#reservationConfirmDelays = getOrCreateCounter("replication", "sqlite_change_log.reservation_confirm_delays", "Snapshot reservations whose confirmation was deferred because the selected source's change-log minimum was later than the backup watermark. Counted once per reservation, by that source.");
 	#latestLagReportCommitTimeMs = 0;
-	#lagReportingDisabled = resolver();
+	#lagReports = resolver();
 	#backupWatermark;
 	#pgPurgedWatermark = "";
 	/**
@@ -330,6 +331,11 @@ var ChangeStreamerImpl = class {
 		this.#purgeLock = initialPurgeLock;
 		this.#autoReset = autoReset;
 		this.#state = new RunningState(this.id, void 0, setTimeoutFn);
+		this.#backupMonitor = opts.backupMonitor;
+		if (this.#backupMonitor) this.#trackBackups(this.#backupMonitor);
+	}
+	async #trackBackups(monitor) {
+		for await (const { watermark } of monitor.backups()) this.#trackBackupWatermark(watermark);
 	}
 	/**
 	* Whether the legacy Postgres change log is enabled, which is determined
@@ -342,7 +348,7 @@ var ChangeStreamerImpl = class {
 		this.#lc.info?.("starting change stream");
 		this.#forwarder.startProgressMonitor();
 		const lagReportInit = await this.#source.startLagReporter();
-		this.#lagReportingDisabled.resolve(!lagReportInit);
+		this.#lagReports.resolve(lagReportInit ? { request: () => this.#source.requestLagReport?.() } : void 0);
 		if (lagReportInit) this.#latestLagReportCommitTimeMs = lagReportInit.firstCommitTimeMs;
 		const storer = this.#storer;
 		let takeoverPending = storer !== null;
@@ -372,13 +378,16 @@ var ChangeStreamerImpl = class {
 				if (this.#state.resetBackoff() > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS) this.#replicationStatusPublisher.publish(this.#lc, "Replicating", `Replicating from ${lastWatermark}`);
 				watermark = null;
 				this.#acker.reset(stream.acks, lastWatermark);
-				for await (const change of stream.changes) {
+				streamLoop: for await (const change of stream.changes) {
 					this.#acker.trackDownstream(change);
 					const [type, msg] = change;
-					if (type === "status" && watermark === null && this.#mayRestart()) break;
 					switch (type) {
 						case "status":
-							if (watermark === null) storer?.status(change[2].watermark);
+							if (msg.syncBackupThrough) this.#backupMonitor?.requestSync(msg.syncBackupThrough);
+							if (watermark === null) {
+								if (this.#mayRestart()) break streamLoop;
+								storer?.status(change[2].watermark);
+							}
 							if (msg.lagReport && msg.lagReport.lastTimings.commitTimeMs >= this.#latestLagReportCommitTimeMs) {
 								this.#latestLagReportCommitTimeMs = msg.lagReport.lastTimings.commitTimeMs;
 								this.#forwarder.sendStatus({
@@ -478,7 +487,7 @@ var ChangeStreamerImpl = class {
 		const lc = this.#lc.withContext("subscriber", id);
 		const subscriber = new Subscriber(protocolVersion, id, mode, watermark, downstream, {
 			wsBatched,
-			lagReportingDisabled: this.#lagReportingDisabled.promise,
+			lagReports: this.#lagReports.promise,
 			drainable: mode === "serving" && ctx.drainable,
 			...this.#backfills.subscriberOptions(lc, ctx)
 		});
@@ -616,7 +625,17 @@ var ChangeStreamerImpl = class {
 	drain() {
 		return this.#drainTracker.drain();
 	}
-	trackBackupWatermark(watermark) {
+	/**
+	* Tracks the watermark up to which the replica has been backed up. This
+	* serves two purposes:
+	* - It serves as the maximum watermark up to which the change-log can be
+	*   purged. Note that the change-streamer also takes into account the
+	*   positions of its subscribers and snapshot reservations when purging
+	*   changes in the change-log.
+	* - In RMv2, this ACK's the upstream change-source to allow its buffer of
+	*   changes (e.g. the replication_slot) to advance.
+	*/
+	#trackBackupWatermark(watermark) {
 		this.#backupWatermark = watermark;
 		this.#acker.trackBackup(watermark);
 		this.#requestSQLitePurge("deferred");
@@ -759,7 +778,7 @@ var ChangeStreamerImpl = class {
 	async stop(err) {
 		this.#state.stop(this.#lc, err);
 		this.#drainTracker.stop();
-		this.#lagReportingDisabled.resolve(false);
+		this.#lagReports.resolve({ request: () => {} });
 		this.#stream?.changes.cancel();
 		this.#purgeScheduler?.stop();
 		this.#comparator?.stop();