@rocicorp/zero 0.26.1 → 0.26.2-canary.1

package/out/zero-cache/src/services/change-streamer/backup-monitor.js

@@ -1,174 +1,168 @@
-import parsePrometheusTextFormat from "parse-prometheus-text-format";
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
-import { Subscription } from "../../types/subscription.js";
 import { RunningState } from "../running-state.js";
-const CHECK_INTERVAL_MS = 6e4;
-const MIN_CLEANUP_DELAY_MS = 3e4;
-class BackupMonitor {
-  id = "backup-monitor";
-  #lc;
-  #backupURL;
-  #metricsEndpoint;
-  #changeStreamer;
-  #state = new RunningState(this.id);
-  #reservations = /* @__PURE__ */ new Map();
-  #watermarks = /* @__PURE__ */ new Map();
-  #lastWatermark = "";
-  #cleanupDelayMs;
-  #checkMetricsTimer;
-  constructor(lc, backupURL, metricsEndpoint, changeStreamer, initialCleanupDelayMs) {
-    this.#lc = lc.withContext("component", this.id);
-    this.#backupURL = backupURL;
-    this.#metricsEndpoint = metricsEndpoint;
-    this.#changeStreamer = changeStreamer;
-    this.#cleanupDelayMs = Math.max(
-      initialCleanupDelayMs,
-      MIN_CLEANUP_DELAY_MS
-      // purely for peace of mind
-    );
-    this.#lc.info?.(
-      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`
-    );
-  }
-  run() {
-    this.#lc.info?.(
-      `monitoring backups at ${this.#metricsEndpoint} with ${this.#cleanupDelayMs} ms cleanup delay`
-    );
-    this.#checkMetricsTimer = setInterval(
-      this.checkWatermarksAndScheduleCleanup,
-      CHECK_INTERVAL_MS
-    );
-    return this.#state.stopped();
-  }
-  startSnapshotReservation(taskID) {
-    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);
-    this.#reservations.get(taskID)?.sub.cancel();
-    const sub = Subscription.create({
-      // If the reservation still exists when the connection closes
-      // (e.g. subscriber crashed), clean it up without updating the
-      // cleanup delay.
-      cleanup: () => this.endReservation(taskID, false)
-    });
-    this.#reservations.set(taskID, { start: /* @__PURE__ */ new Date(), sub });
-    void this.#changeStreamer.getChangeLogState().then((changeLogState) => {
-      sub.push([
-        "status",
-        { tag: "status", backupURL: this.#backupURL, ...changeLogState }
-      ]);
-    }).catch((e) => {
-      this.#lc.warn?.(`failing snapshot reservation`, e);
-      sub.fail(e);
-    });
-    return sub;
-  }
-  endReservation(taskID, updateCleanupDelay = true) {
-    const res = this.#reservations.get(taskID);
-    if (res === void 0) {
-      return;
-    }
-    this.#reservations.delete(taskID);
-    const { start, sub } = res;
-    sub.cancel();
-    if (updateCleanupDelay) {
-      const duration = Date.now() - start.getTime();
-      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);
-      if (duration > this.#cleanupDelayMs) {
-        this.#cleanupDelayMs = duration;
-        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);
-      }
-    }
-  }
-  // Exported for testing
-  checkWatermarksAndScheduleCleanup = async () => {
-    try {
-      await this.#checkWatermarks();
-    } catch (e) {
-      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);
-    }
-    try {
-      this.#scheduleCleanup();
-    } catch (e) {
-      this.#lc.warn?.(`error scheduling cleanup`, e);
-    }
-  };
-  async *#fetchWatermarks() {
-    const metricsEndpoint = this.#metricsEndpoint;
-    const signal = this.#state.signal;
-    let resp;
-    try {
-      resp = await fetch(metricsEndpoint, { signal });
-    } catch (e) {
-      if (signal.aborted) {
-        return;
-      }
-      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);
-      return;
-    }
-    if (!resp.ok) {
-      this.#lc.warn?.(
-        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`
-      );
-      return;
-    }
-    const families = parsePrometheusTextFormat(await resp.text());
-    for (const family of families) {
-      if (family.type === "GAUGE" && family.name === "litestream_replica_progress") {
-        for (const metric of family.metrics) {
-          const watermark = metric.labels?.watermark;
-          const name = metric.labels?.name;
-          const time = new Date(parseFloat(metric.value) * 1e3);
-          if (watermark) {
-            yield { watermark, time, name };
-          }
-        }
-      }
-    }
-  }
-  async #checkWatermarks() {
-    for await (const { watermark, name, time } of this.#fetchWatermarks()) {
-      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {
-        this.#lc.info?.(
-          `replicated watermark=${watermark} to ${name} at ${time.toISOString()}.`
-        );
-        this.#watermarks.set(watermark, time);
-      }
-    }
-  }
-  #scheduleCleanup() {
-    if (this.#reservations.size > 0) {
-      this.#lc.info?.(
-        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`
-      );
-      return;
-    }
-    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;
-    let maxWatermark = "";
-    for (const [watermark, backupTime] of this.#watermarks.entries()) {
-      if (backupTime.getTime() <= latestCleanupTime && watermark > maxWatermark) {
-        maxWatermark = watermark;
-      }
-    }
-    if (maxWatermark.length) {
-      this.#changeStreamer.scheduleCleanup(maxWatermark);
-      for (const watermark of this.#watermarks.keys()) {
-        if (watermark <= maxWatermark) {
-          this.#watermarks.delete(watermark);
-        }
-      }
-      this.#lastWatermark = maxWatermark;
-    }
-  }
-  stop() {
-    clearInterval(this.#checkMetricsTimer);
-    for (const { sub } of this.#reservations.values()) {
-      sub.cancel();
-    }
-    this.#state.stop(this.#lc);
-    return promiseVoid;
-  }
-}
-export {
-  BackupMonitor,
-  CHECK_INTERVAL_MS
+import { Subscription } from "../../types/subscription.js";
+import parsePrometheusTextFormat from "parse-prometheus-text-format";
+//#region ../zero-cache/src/services/change-streamer/backup-monitor.ts
+var CHECK_INTERVAL_MS = 6e4;
+var MIN_CLEANUP_DELAY_MS = 3e4;
+/**
+* The BackupMonitor polls the litestream "/metrics" endpoint to track the
+* watermark (label) value of the `litestream_replica_progress` gauge and
+* schedules cleanup of change log entries that can be purged as a result.
+*
+* See: https://github.com/rocicorp/litestream/pull/3
+*
+* Note that change log entries cannot simply be purged as soon as they
+* have been applied and backed up by litestream. Consider the case in which
+* litestream backs up new wal segments every minute, but it takes 5 minutes
+* to restore a replica: if a zero-cache starts restoring a replica at
+* minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
+* purging changelog records as soon as those watermarks are replicated would
+* result in the zero-cache not being able to catch up from minute 0 once it
+* has finished restoring the replica.
+*
+* The `/snapshot` reservation protocol is used to prevent premature change
+* log cleanup:
+* - Clients restoring a snapshot initiate a `/snapshot` request and hold that
+*   request open while it restores its snapshot, prepares it, and
+*   starts its subscription to the change stream. During this time, no
+*   cleanups are scheduled.
+* - When the subscription is started, the interval since the beginning of
+*   of the reservation is tracked to increase the background cleanup delay
+*   interval if needed. The reservation is ended (and request closed), and
+*   cleanup scheduling is resumed with the current delay interval.
+*
+* Note that the reservation request is the primary mechanism by which
+* premature change log cleanup is prevented. The cleanup delay interval is
+* a secondary safeguard.
+*/
+var BackupMonitor = class {
+	id = "backup-monitor";
+	#lc;
+	#backupURL;
+	#metricsEndpoint;
+	#changeStreamer;
+	#state = new RunningState(this.id);
+	#reservations = /* @__PURE__ */ new Map();
+	#watermarks = /* @__PURE__ */ new Map();
+	#lastWatermark = "";
+	#cleanupDelayMs;
+	#checkMetricsTimer;
+	constructor(lc, backupURL, metricsEndpoint, changeStreamer, initialCleanupDelayMs) {
+		this.#lc = lc.withContext("component", this.id);
+		this.#backupURL = backupURL;
+		this.#metricsEndpoint = metricsEndpoint;
+		this.#changeStreamer = changeStreamer;
+		this.#cleanupDelayMs = Math.max(initialCleanupDelayMs, MIN_CLEANUP_DELAY_MS);
+		this.#lc.info?.(`backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`);
+	}
+	run() {
+		this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint} with ${this.#cleanupDelayMs} ms cleanup delay`);
+		this.#checkMetricsTimer = setInterval(this.checkWatermarksAndScheduleCleanup, CHECK_INTERVAL_MS);
+		return this.#state.stopped();
+	}
+	startSnapshotReservation(taskID) {
+		this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);
+		this.#reservations.get(taskID)?.sub.cancel();
+		const sub = Subscription.create({ cleanup: () => this.endReservation(taskID, false) });
+		this.#reservations.set(taskID, {
+			start: /* @__PURE__ */ new Date(),
+			sub
+		});
+		this.#changeStreamer.getChangeLogState().then((changeLogState) => {
+			sub.push(["status", {
+				tag: "status",
+				backupURL: this.#backupURL,
+				...changeLogState
+			}]);
+		}).catch((e) => {
+			this.#lc.warn?.(`failing snapshot reservation`, e);
+			sub.fail(e);
+		});
+		return sub;
+	}
+	endReservation(taskID, updateCleanupDelay = true) {
+		const res = this.#reservations.get(taskID);
+		if (res === void 0) return;
+		this.#reservations.delete(taskID);
+		const { start, sub } = res;
+		sub.cancel();
+		if (updateCleanupDelay) {
+			const duration = Date.now() - start.getTime();
+			this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);
+			if (duration > this.#cleanupDelayMs) {
+				this.#cleanupDelayMs = duration;
+				this.#lc.info?.(`increased cleanup delay to ${duration} ms`);
+			}
+		}
+	}
+	checkWatermarksAndScheduleCleanup = async () => {
+		try {
+			await this.#checkWatermarks();
+		} catch (e) {
+			this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);
+		}
+		try {
+			this.#scheduleCleanup();
+		} catch (e) {
+			this.#lc.warn?.(`error scheduling cleanup`, e);
+		}
+	};
+	async *#fetchWatermarks() {
+		const metricsEndpoint = this.#metricsEndpoint;
+		const signal = this.#state.signal;
+		let resp;
+		try {
+			resp = await fetch(metricsEndpoint, { signal });
+		} catch (e) {
+			if (signal.aborted) return;
+			this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);
+			return;
+		}
+		if (!resp.ok) {
+			this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`);
+			return;
+		}
+		const families = parsePrometheusTextFormat(await resp.text());
+		for (const family of families) if (family.type === "GAUGE" && family.name === "litestream_replica_progress") for (const metric of family.metrics) {
+			const watermark = metric.labels?.watermark;
+			const name = metric.labels?.name;
+			const time = /* @__PURE__ */ new Date(parseFloat(metric.value) * 1e3);
+			if (watermark) yield {
+				watermark,
+				time,
+				name
+			};
+		}
+	}
+	async #checkWatermarks() {
+		for await (const { watermark, name, time } of this.#fetchWatermarks()) if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {
+			this.#lc.info?.(`replicated watermark=${watermark} to ${name} at ${time.toISOString()}.`);
+			this.#watermarks.set(watermark, time);
+		}
+	}
+	#scheduleCleanup() {
+		if (this.#reservations.size > 0) {
+			this.#lc.info?.(`watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`);
+			return;
+		}
+		const latestCleanupTime = Date.now() - this.#cleanupDelayMs;
+		let maxWatermark = "";
+		for (const [watermark, backupTime] of this.#watermarks.entries()) if (backupTime.getTime() <= latestCleanupTime && watermark > maxWatermark) maxWatermark = watermark;
+		if (maxWatermark.length) {
+			this.#changeStreamer.scheduleCleanup(maxWatermark);
+			for (const watermark of this.#watermarks.keys()) if (watermark <= maxWatermark) this.#watermarks.delete(watermark);
+			this.#lastWatermark = maxWatermark;
+		}
+	}
+	stop() {
+		clearInterval(this.#checkMetricsTimer);
+		for (const { sub } of this.#reservations.values()) sub.cancel();
+		this.#state.stop(this.#lc);
+		return promiseVoid;
+	}
 };
-//# sourceMappingURL=backup-monitor.js.map
+//#endregion
+export { BackupMonitor };
+
+//# sourceMappingURL=backup-monitor.js.map

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60_000;\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The BackupMonitor polls the litestream \"/metrics\" endpoint to track the\n * watermark (label) value of the `litestream_replica_progress` gauge and\n * schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n */\nexport class BackupMonitor implements Service {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  #lastWatermark: string = '';\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    // Note: the Subscription must be returned immediately so that the\n    //       websocket can begin sending liveness pings.\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n      }\n    }\n  }\n\n  #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    let maxWatermark = '';\n    for (const [watermark, backupTime] of this.#watermarks.entries()) {\n      if (\n        backupTime.getTime() <= latestCleanupTime &&\n        watermark > maxWatermark\n      ) {\n        maxWatermark = watermark;\n      }\n    }\n    if (maxWatermark.length) {\n      this.#changeStreamer.scheduleCleanup(maxWatermark);\n      for (const watermark of this.#watermarks.keys()) {\n        if (watermark <= maxWatermark) {\n          this.#watermarks.delete(watermark);\n        }\n      }\n      this.#lastWatermark = maxWatermark;\n    }\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"names":[],"mappings":";;;;AASO,MAAM,oBAAoB;AACjC,MAAM,uBAAuB;AAsCtB,MAAM,cAAiC;AAAA,EACnC,KAAK;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,aAAa,KAAK,EAAE;AAAA,EAEjC,oCAAoB,IAAA;AAAA,EACpB,kCAAkB,IAAA;AAAA,EAE3B,iBAAyB;AAAA,EACzB;AAAA,EACA;AAAA,EAEA,YACE,IACA,WACA,iBACA,gBACA,uBACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;AAC9C,SAAK,aAAa;AAClB,SAAK,mBAAmB;AACxB,SAAK,kBAAkB;AACvB,SAAK,kBAAkB,KAAK;AAAA,MAC1B;AAAA,MACA;AAAA;AAAA,IAAA;AAGF,SAAK,IAAI;AAAA,MACP,0BAA0B,qBAAqB;AAAA,IAAA;AAAA,EAEnD;AAAA,EAEA,MAAqB;AACnB,SAAK,IAAI;AAAA,MACP,yBAAyB,KAAK,gBAAgB,SACzC,KAAK,eAAe;AAAA,IAAA;AAE3B,SAAK,qBAAqB;AAAA,MACxB,KAAK;AAAA,MACL;AAAA,IAAA;AAEF,WAAO,KAAK,OAAO,QAAA;AAAA,EACrB;AAAA,EAEA,yBAAyB,QAA+C;AACtE,SAAK,IAAI,OAAO,oCAAoC,MAAM,YAAY;AAEtE,SAAK,cAAc,IAAI,MAAM,GAAG,IAAI,OAAA;AAEpC,UAAM,MAAM,aAAa,OAAwB;AAAA;AAAA;AAAA;AAAA,MAI/C,SAAS,MAAM,KAAK,eAAe,QAAQ,KAAK;AAAA,IAAA,CACjD;AACD,SAAK,cAAc,IAAI,QAAQ,EAAC,OAAO,oBAAI,QAAQ,KAAI;AAGvD,SAAK,KAAK,gBACP,kBAAA,EACA,KAAK,CAAA,mBAAkB;AACtB,UAAI,KAAK;AAAA,QACP;AAAA,QACA,EAAC,KAAK,UAAU,WAAW,KAAK,YAAY,GAAG,eAAA;AAAA,MAAc,CAC9D;AAAA,IACH,CAAC,EACA,MAAM,CAAA,MAAK;AACV,WAAK,IAAI,OAAO,gCAAgC,CAAC;AACjD,UAAI,KAAK,CAAC;AAAA,IACZ,CAAC;AACH,WAAO;AAAA,EACT;AAAA,EAEA,eAAe,QAAgB,qBAAqB,MAAM;AACxD,UAAM,MAAM,KAAK,cAAc,IAAI,MAAM;AACzC,QAAI,QAAQ,QAAW;AACrB;AAAA,IACF;AACA,SAAK,cAAc,OAAO,MAAM;AAChC,UAAM,EAAC,OAAO,IAAA,IAAO;AACrB,QAAI,OAAA;AAEJ,QAAI,oBAAoB;AACtB,YAAM,WAAW,KAAK,IAAA,IAAQ,MAAM,QAAA;AACpC,WAAK,IAAI,OAAO,2BAA2B,MAAM,OAAO,QAAQ,KAAK;AACrE,UAAI,WAAW,KAAK,iBAAiB;AACnC,aAAK,kBAAkB;AACvB,aAAK,IAAI,OAAO,8BAA8B,QAAQ,KAAK;AAAA,MAC7D;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGS,oCAAoC,YAAY;AACvD,QAAI;AACF,YAAM,KAAK,iBAAA;AAAA,IACb,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,8BAA8B,KAAK,gBAAgB,IAAI,CAAC;AAAA,IAC1E;AACA,QAAI;AACF,WAAK,iBAAA;AAAA,IACP,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,4BAA4B,CAAC;AAAA,IAC/C;AAAA,EACF;AAAA,EAEA,OAAO,mBAIJ;AACD,UAAM,kBAAkB,KAAK;AAC7B,UAAM,SAAS,KAAK,OAAO;AAC3B,QAAI;AACJ,QAAI;AACF,aAAO,MAAM,MAAM,iBAAiB,EAAC,QAAO;AAAA,IAC9C,SAAS,GAAG;AACV,UAAI,OAAO,SAAS;AAElB;AAAA,MACF;AAGA,WAAK,IAAI,OAAO,8BAA8B,KAAK,gBAAgB,IAAI,CAAC;AACxE;AAAA,IACF;AACA,QAAI,CAAC,KAAK,IAAI;AACZ,WAAK,IAAI;AAAA,QACP,8BAA8B,KAAK,gBAAgB,KAAK,MAAM,KAAK,MAAM;AAAA,MAAA;AAE3E;AAAA,IACF;AAEA,UAAM,WAAW,0BAA0B,MAAM,KAAK,MAAM;AAC5D,eAAW,UAAU,UAAU;AAC7B,UACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAChB;AACA,mBAAW,UAAU,OAAO,SAAS;AACnC,gBAAM,YAAY,OAAO,QAAQ;AACjC,gBAAM,OAAO,OAAO,QAAQ;AAC5B,gBAAM,OAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;AAErD,cAAI,WAAW;AACb,kBAAM,EAAC,WAAW,MAAM,KAAA;AAAA,UAC1B;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB;AACvB,qBAAiB,EAAC,WAAW,MAAM,UAAS,KAAK,oBAAoB;AACnE,UAAI,YAAY,KAAK,kBAAkB,CAAC,KAAK,YAAY,IAAI,SAAS,GAAG;AACvE,aAAK,IAAI;AAAA,UACP,wBAAwB,SAAS,OAAO,IAAI,OACnC,KAAK,aAAa;AAAA,QAAA;AAE7B,aAAK,YAAY,IAAI,WAAW,IAAI;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,mBAAmB;AACjB,QAAI,KAAK,cAAc,OAAO,GAAG;AAC/B,WAAK,IAAI;AAAA,QACP,6CAA6C,CAAC,GAAG,KAAK,cAAc,KAAA,CAAM,CAAC;AAAA,MAAA;AAE7E;AAAA,IACF;AACA,UAAM,oBAAoB,KAAK,IAAA,IAAQ,KAAK;AAC5C,QAAI,eAAe;AACnB,eAAW,CAAC,WAAW,UAAU,KAAK,KAAK,YAAY,WAAW;AAChE,UACE,WAAW,QAAA,KAAa,qBACxB,YAAY,cACZ;AACA,uBAAe;AAAA,MACjB;AAAA,IACF;AACA,QAAI,aAAa,QAAQ;AACvB,WAAK,gBAAgB,gBAAgB,YAAY;AACjD,iBAAW,aAAa,KAAK,YAAY,KAAA,GAAQ;AAC/C,YAAI,aAAa,cAAc;AAC7B,eAAK,YAAY,OAAO,SAAS;AAAA,QACnC;AAAA,MACF;AACA,WAAK,iBAAiB;AAAA,IACxB;AAAA,EACF;AAAA,EAEA,OAAsB;AACpB,kBAAc,KAAK,kBAAkB;AACrC,eAAW,EAAC,IAAA,KAAQ,KAAK,cAAc,UAAU;AAK/C,UAAI,OAAA;AAAA,IACN;AACA,SAAK,OAAO,KAAK,KAAK,GAAG;AACzB,WAAO;AAAA,EACT;AACF;"}
+{"version":3,"file":"backup-monitor.js","names":["#lc","#backupURL","#metricsEndpoint","#changeStreamer","#state","#reservations","#watermarks","#cleanupDelayMs","#checkMetricsTimer","#checkWatermarks","#scheduleCleanup","#fetchWatermarks","#lastWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60_000;\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The BackupMonitor polls the litestream \"/metrics\" endpoint to track the\n * watermark (label) value of the `litestream_replica_progress` gauge and\n * schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n */\nexport class BackupMonitor implements Service {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  #lastWatermark: string = '';\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    // Note: the Subscription must be returned immediately so that the\n    //       websocket can begin sending liveness pings.\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n      }\n    }\n  }\n\n  #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    let maxWatermark = '';\n    for (const [watermark, backupTime] of this.#watermarks.entries()) {\n      if (\n        backupTime.getTime() <= latestCleanupTime &&\n        watermark > maxWatermark\n      ) {\n        maxWatermark = watermark;\n      }\n    }\n    if (maxWatermark.length) {\n      this.#changeStreamer.scheduleCleanup(maxWatermark);\n      for (const watermark of this.#watermarks.keys()) {\n        if (watermark <= maxWatermark) {\n          this.#watermarks.delete(watermark);\n        }\n      }\n      this.#lastWatermark = maxWatermark;\n    }\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;AASA,IAAa,oBAAoB;AACjC,IAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsC7B,IAAa,gBAAb,MAA8C;CAC5C,KAAc;CACd;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,GAAG;CAE3C,gCAAyB,IAAI,KAA0B;CACvD,8BAAuB,IAAI,KAAmB;CAE9C,iBAAyB;CACzB;CACA;CAEA,YACE,IACA,WACA,iBACA,gBACA,uBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,KAAK,GAAG;AAC/C,QAAA,YAAkB;AAClB,QAAA,kBAAwB;AACxB,QAAA,iBAAuB;AACvB,QAAA,iBAAuB,KAAK,IAC1B,uBACA,qBACD;AAED,QAAA,GAAS,OACP,0BAA0B,sBAAsB,4BACjD;;CAGH,MAAqB;AACnB,QAAA,GAAS,OACP,yBAAyB,MAAA,gBAAsB,QAC1C,MAAA,eAAqB,mBAC3B;AACD,QAAA,oBAA0B,YACxB,KAAK,mCACL,kBACD;AACD,SAAO,MAAA,MAAY,SAAS;;CAG9B,yBAAyB,QAA+C;AACtE,QAAA,GAAS,OAAO,oCAAoC,OAAO,YAAY;AAEvE,QAAA,aAAmB,IAAI,OAAO,EAAE,IAAI,QAAQ;EAE5C,MAAM,MAAM,aAAa,OAAwB,EAI/C,eAAe,KAAK,eAAe,QAAQ,MAAM,EAClD,CAAC;AACF,QAAA,aAAmB,IAAI,QAAQ;GAAC,uBAAO,IAAI,MAAM;GAAE;GAAI,CAAC;AAGnD,QAAA,eACF,mBAAmB,CACnB,MAAK,mBAAkB;AACtB,OAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,MAAA;IAAiB,GAAG;IAAe,CAC/D,CAAC;IACF,CACD,OAAM,MAAK;AACV,SAAA,GAAS,OAAO,gCAAgC,EAAE;AAClD,OAAI,KAAK,EAAE;IACX;AACJ,SAAO;;CAGT,eAAe,QAAgB,qBAAqB,MAAM;EACxD,MAAM,MAAM,MAAA,aAAmB,IAAI,OAAO;AAC1C,MAAI,QAAQ,KAAA,EACV;AAEF,QAAA,aAAmB,OAAO,OAAO;EACjC,MAAM,EAAC,OAAO,QAAO;AACrB,MAAI,QAAQ;AAEZ,MAAI,oBAAoB;GACtB,MAAM,WAAW,KAAK,KAAK,GAAG,MAAM,SAAS;AAC7C,SAAA,GAAS,OAAO,2BAA2B,OAAO,MAAM,SAAS,KAAK;AACtE,OAAI,WAAW,MAAA,gBAAsB;AACnC,UAAA,iBAAuB;AACvB,UAAA,GAAS,OAAO,8BAA8B,SAAS,KAAK;;;;CAMlE,oCAA6C,YAAY;AACvD,MAAI;AACF,SAAM,MAAA,iBAAuB;WACtB,GAAG;AACV,SAAA,GAAS,OAAO,8BAA8B,MAAA,mBAAyB,EAAE;;AAE3E,MAAI;AACF,SAAA,iBAAuB;WAChB,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;;;CAIlD,QAAA,kBAIG;EACD,MAAM,kBAAkB,MAAA;EACxB,MAAM,SAAS,MAAA,MAAY;EAC3B,IAAI;AACJ,MAAI;AACF,UAAO,MAAM,MAAM,iBAAiB,EAAC,QAAO,CAAC;WACtC,GAAG;AACV,OAAI,OAAO,QAET;AAIF,SAAA,GAAS,OAAO,8BAA8B,MAAA,mBAAyB,EAAE;AACzE;;AAEF,MAAI,CAAC,KAAK,IAAI;AACZ,SAAA,GAAS,OACP,8BAA8B,MAAA,gBAAsB,IAAI,MAAM,KAAK,MAAM,GAC1E;AACD;;EAGF,MAAM,WAAW,0BAA0B,MAAM,KAAK,MAAM,CAAC;AAC7D,OAAK,MAAM,UAAU,SACnB,KACE,OAAO,SAAS,WAChB,OAAO,SAAS,8BAEhB,MAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,MAAM,GAAG,IAAK;AAEtD,OAAI,UACF,OAAM;IAAC;IAAW;IAAM;IAAK;;;CAOvC,OAAA,kBAAyB;AACvB,aAAW,MAAM,EAAC,WAAW,MAAM,UAAS,MAAA,iBAAuB,CACjE,KAAI,YAAY,MAAA,iBAAuB,CAAC,MAAA,WAAiB,IAAI,UAAU,EAAE;AACvE,SAAA,GAAS,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,aAAa,CAAC,GAC7B;AACD,SAAA,WAAiB,IAAI,WAAW,KAAK;;;CAK3C,mBAAmB;AACjB,MAAI,MAAA,aAAmB,OAAO,GAAG;AAC/B,SAAA,GAAS,OACP,6CAA6C,CAAC,GAAG,MAAA,aAAmB,MAAM,CAAC,GAC5E;AACD;;EAEF,MAAM,oBAAoB,KAAK,KAAK,GAAG,MAAA;EACvC,IAAI,eAAe;AACnB,OAAK,MAAM,CAAC,WAAW,eAAe,MAAA,WAAiB,SAAS,CAC9D,KACE,WAAW,SAAS,IAAI,qBACxB,YAAY,aAEZ,gBAAe;AAGnB,MAAI,aAAa,QAAQ;AACvB,SAAA,eAAqB,gBAAgB,aAAa;AAClD,QAAK,MAAM,aAAa,MAAA,WAAiB,MAAM,CAC7C,KAAI,aAAa,aACf,OAAA,WAAiB,OAAO,UAAU;AAGtC,SAAA,gBAAsB;;;CAI1B,OAAsB;AACpB,gBAAc,MAAA,kBAAwB;AACtC,OAAK,MAAM,EAAC,SAAQ,MAAA,aAAmB,QAAQ,CAK7C,KAAI,QAAQ;AAEd,QAAA,MAAY,KAAK,MAAA,GAAS;AAC1B,SAAO"}