@rocicorp/zero 1.7.0 → 1.8.0-canary.1

package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.js

@@ -0,0 +1,140 @@
+import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { getOrCreateCounter, getOrCreateGauge } from "../../observability/metrics.js";
+import { RunningState } from "../running-state.js";
+import { Subscription } from "../../types/subscription.js";
+//#region ../zero-cache/src/services/change-streamer/vfs-backup-monitor.ts
+var MIN_CLEANUP_DELAY_MS = 3e4;
+/**
+* Monitors a Litestream v0.5.x backup by reading Zero's replication state from
+* the backup itself through the Litestream SQLite VFS.
+*/
+var VfsBackupMonitor = class {
+	id = "vfs-backup-monitor";
+	#lc;
+	#backupURL;
+	#changeStreamer;
+	#source;
+	#probeIntervalMs;
+	#state = new RunningState(this.id);
+	#reservations = /* @__PURE__ */ new Map();
+	#watermarks = /* @__PURE__ */ new Map();
+	#purgesBlocked = getOrCreateCounter("replica", "purge_blocked", { description: "Number of change-log purges blocked because the actual backup watermark could not be read through the Litestream VFS." });
+	#lastWatermark = "";
+	#latestBackupWatermark;
+	#cleanupDelayMs;
+	#checkWatermarkTimer;
+	constructor(lc, backupURL, changeStreamer, initialCleanupDelayMs, probeIntervalMs, source) {
+		this.#lc = lc.withContext("component", this.id);
+		this.#backupURL = backupURL;
+		this.#changeStreamer = changeStreamer;
+		this.#source = source;
+		this.#probeIntervalMs = probeIntervalMs;
+		this.#cleanupDelayMs = Math.max(initialCleanupDelayMs, MIN_CLEANUP_DELAY_MS);
+	}
+	run() {
+		this.#lc.info?.(`monitoring v5 backups at ${this.#backupURL} with ${this.#cleanupDelayMs} ms cleanup delay`);
+		this.#checkWatermarkTimer = setInterval(this.checkWatermarkAndScheduleCleanup, this.#probeIntervalMs);
+		this.#initBackupLagMetric();
+		return this.#state.stopped();
+	}
+	startSnapshotReservation(taskID) {
+		this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);
+		this.#reservations.get(taskID)?.sub.cancel();
+		const sub = Subscription.create({ cleanup: () => this.endReservation(taskID, false) });
+		this.#reservations.set(taskID, {
+			start: /* @__PURE__ */ new Date(),
+			sub
+		});
+		this.#changeStreamer.getChangeLogState().then((changeLogState) => {
+			sub.push(["status", {
+				tag: "status",
+				backupURL: this.#backupURL,
+				...changeLogState
+			}]);
+		}).catch((e) => {
+			this.#lc.warn?.(`failing snapshot reservation`, e);
+			sub.fail(e);
+		});
+		return sub;
+	}
+	endReservation(taskID, updateCleanupDelay = true) {
+		const res = this.#reservations.get(taskID);
+		if (res === void 0) return;
+		this.#reservations.delete(taskID);
+		const { start, sub } = res;
+		sub.cancel();
+		if (updateCleanupDelay) {
+			const duration = Date.now() - start.getTime();
+			this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);
+			if (duration > this.#cleanupDelayMs) {
+				this.#cleanupDelayMs = duration;
+				this.#lc.info?.(`increased cleanup delay to ${duration} ms`);
+			}
+		}
+	}
+	checkWatermarkAndScheduleCleanup = async () => {
+		try {
+			await this.#checkWatermark();
+		} catch (e) {
+			this.#purgesBlocked.add(1, { reason: "vfs-probe-failed" });
+			this.#lc.warn?.("unable to read backup watermark through Litestream VFS. skipping change-log cleanup", e);
+			return;
+		}
+		this.#scheduleCleanup();
+	};
+	async #checkWatermark() {
+		const watermark = await this.#source.readWatermark();
+		this.#latestBackupWatermark = watermark;
+		if (watermark.watermark > this.#lastWatermark && !this.#watermarks.has(watermark.watermark)) {
+			this.#lc.info?.(`observed backup watermark=${watermark.watermark} through Litestream VFS at ${new Date(watermark.observedAtMs).toISOString()}`, {
+				writeTimeMs: watermark.writeTimeMs,
+				txid: watermark.txid,
+				lagSeconds: watermark.lagSeconds
+			});
+			this.#watermarks.set(watermark.watermark, watermark);
+		}
+	}
+	#scheduleCleanup() {
+		if (this.#reservations.size > 0) {
+			this.#lc.info?.(`watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`);
+			return;
+		}
+		const latestConfirmedWatermark = this.#latestBackupWatermark?.watermark;
+		if (latestConfirmedWatermark === void 0) return;
+		const maxWatermark = this.#maxWatermarkUpTo(Date.now() - this.#cleanupDelayMs, latestConfirmedWatermark);
+		if (maxWatermark.length === 0) return;
+		this.#changeStreamer.scheduleCleanup(maxWatermark);
+		for (const watermark of this.#watermarks.keys()) if (watermark <= maxWatermark) this.#watermarks.delete(watermark);
+		this.#lastWatermark = maxWatermark;
+	}
+	#maxWatermarkUpTo(cutoff, latestConfirmedWatermark) {
+		let max = "";
+		for (const [watermark, backupWatermark] of this.#watermarks) if (watermark <= latestConfirmedWatermark && backupWatermark.observedAtMs <= cutoff && watermark > max) max = watermark;
+		return max;
+	}
+	stop() {
+		clearInterval(this.#checkWatermarkTimer);
+		for (const { sub } of this.#reservations.values()) sub.cancel();
+		this.#source.close?.();
+		this.#state.stop(this.#lc);
+		return promiseVoid;
+	}
+	#initBackupLagMetric() {
+		getOrCreateGauge("replica", "backup_lag", {
+			description: "Latency from when a change is written to the replica to when it is backed up to litestream.",
+			unit: "millisecond"
+		}).addCallback((o) => {
+			const latestBackupWatermark = this.#latestBackupWatermark;
+			if (latestBackupWatermark?.writeTimeMs === void 0) {
+				this.#lc.warn?.(`no backed up watermarks. unable to report replica.backup_lag`);
+				return;
+			}
+			if (latestBackupWatermark.writeTimeMs === null) return;
+			o.observe(Math.max(0, latestBackupWatermark.observedAtMs - latestBackupWatermark.writeTimeMs));
+		});
+	}
+};
+//#endregion
+export { VfsBackupMonitor };
+
+//# sourceMappingURL=vfs-backup-monitor.js.map

package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-backup-monitor.js","names":["#lc","#backupURL","#changeStreamer","#source","#probeIntervalMs","#state","#reservations","#watermarks","#purgesBlocked","#cleanupDelayMs","#checkWatermarkTimer","#initBackupLagMetric","#checkWatermark","#scheduleCleanup","#latestBackupWatermark","#lastWatermark","#maxWatermarkUpTo"],"sources":["../../../../../../zero-cache/src/services/change-streamer/vfs-backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n} from '../../observability/metrics.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {VfsBackupWatermark} from '../litestream/vfs-watermark-reader.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\nexport interface VfsBackupWatermarkSource {\n  readWatermark(): Promise<VfsBackupWatermark>;\n  close?(): void;\n}\n\n/**\n * Monitors a Litestream v0.5.x backup by reading Zero's replication state from\n * the backup itself through the Litestream SQLite VFS.\n */\nexport class VfsBackupMonitor implements BackupMonitor {\n  readonly id = 'vfs-backup-monitor';\n  readonly #lc: LogContext;\n  readonly #backupURL: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #source: VfsBackupWatermarkSource;\n  readonly #probeIntervalMs: number;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, VfsBackupWatermark>();\n\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup ' +\n      'watermark could not be read through the Litestream VFS.',\n  });\n\n  #lastWatermark: string = '';\n  #latestBackupWatermark: VfsBackupWatermark | undefined;\n  #cleanupDelayMs: number;\n  #checkWatermarkTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    backupURL: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n    probeIntervalMs: number,\n    source: VfsBackupWatermarkSource,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#backupURL = backupURL;\n    this.#changeStreamer = changeStreamer;\n    this.#source = source;\n    this.#probeIntervalMs = probeIntervalMs;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring v5 backups at ${this.#backupURL} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkWatermarkTimer = setInterval(\n      this.checkWatermarkAndScheduleCleanup,\n      this.#probeIntervalMs,\n    );\n    this.#initBackupLagMetric();\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true): void {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel();\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarkAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermark();\n    } catch (e) {\n      this.#purgesBlocked.add(1, {reason: 'vfs-probe-failed'});\n      this.#lc.warn?.(\n        `unable to read backup watermark through Litestream VFS. ` +\n          `skipping change-log cleanup`,\n        e,\n      );\n      return;\n    }\n\n    this.#scheduleCleanup();\n  };\n\n  async #checkWatermark(): Promise<void> {\n    const watermark = await this.#source.readWatermark();\n    this.#latestBackupWatermark = watermark;\n    if (\n      watermark.watermark > this.#lastWatermark &&\n      !this.#watermarks.has(watermark.watermark)\n    ) {\n      this.#lc.info?.(\n        `observed backup watermark=${watermark.watermark} through ` +\n          `Litestream VFS at ${new Date(watermark.observedAtMs).toISOString()}`,\n        {\n          writeTimeMs: watermark.writeTimeMs,\n          txid: watermark.txid,\n          lagSeconds: watermark.lagSeconds,\n        },\n      );\n      this.#watermarks.set(watermark.watermark, watermark);\n    }\n  }\n\n  #scheduleCleanup(): void {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n\n    const latestConfirmedWatermark = this.#latestBackupWatermark?.watermark;\n    if (latestConfirmedWatermark === undefined) {\n      return;\n    }\n\n    const maxWatermark = this.#maxWatermarkUpTo(\n      Date.now() - this.#cleanupDelayMs,\n      latestConfirmedWatermark,\n    );\n    if (maxWatermark.length === 0) {\n      return;\n    }\n\n    this.#changeStreamer.scheduleCleanup(maxWatermark);\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= maxWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#lastWatermark = maxWatermark;\n  }\n\n  #maxWatermarkUpTo(cutoff: number, latestConfirmedWatermark: string): string {\n    let max = '';\n    for (const [watermark, backupWatermark] of this.#watermarks) {\n      if (\n        watermark <= latestConfirmedWatermark &&\n        backupWatermark.observedAtMs <= cutoff &&\n        watermark > max\n      ) {\n        max = watermark;\n      }\n    }\n    return max;\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkWatermarkTimer);\n    for (const {sub} of this.#reservations.values()) {\n      sub.cancel();\n    }\n    this.#source.close?.();\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric(): void {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackupWatermark = this.#latestBackupWatermark;\n      if (latestBackupWatermark?.writeTimeMs === undefined) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      if (latestBackupWatermark.writeTimeMs === null) {\n        return;\n      }\n      o.observe(\n        Math.max(\n          0,\n          latestBackupWatermark.observedAtMs -\n            latestBackupWatermark.writeTimeMs,\n        ),\n      );\n    });\n  }\n}\n"],"mappings":";;;;;AAaA,IAAM,uBAAuB;;;;;AAgB7B,IAAa,mBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,gCAAyB,IAAI,IAAyB;CACtD,8BAAuB,IAAI,IAAgC;CAE3D,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,wHAEJ,CAAC;CAED,iBAAyB;CACzB;CACA;CACA;CAEA,YACE,IACA,WACA,gBACA,uBACA,iBACA,QACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,aAAa;EAClB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,mBAAmB;EACxB,KAAKK,kBAAkB,KAAK,IAC1B,uBACA,oBACF;CACF;CAEA,MAAqB;EACnB,KAAKT,IAAI,OACP,4BAA4B,KAAKC,WAAW,QACvC,KAAKQ,gBAAgB,kBAC5B;EACA,KAAKC,uBAAuB,YAC1B,KAAK,kCACL,KAAKN,gBACP;EACA,KAAKO,qBAAqB;EAC1B,OAAO,KAAKN,OAAO,QAAQ;CAC7B;CAEA,yBAAyB,QAA+C;EACtE,KAAKL,IAAI,OAAO,oCAAoC,OAAO,WAAW;EACtE,KAAKM,cAAc,IAAI,MAAM,GAAG,IAAI,OAAO;EAE3C,MAAM,MAAM,aAAa,OAAwB,EAC/C,eAAe,KAAK,eAAe,QAAQ,KAAK,EAClD,CAAC;EACD,KAAKA,cAAc,IAAI,QAAQ;GAAC,uBAAO,IAAI,KAAK;GAAG;EAAG,CAAC;EAEvD,KAAUJ,gBACP,kBAAkB,EAClB,MAAK,mBAAkB;GACtB,IAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,KAAKD;IAAY,GAAG;GAAc,CAC/D,CAAC;EACH,CAAC,EACA,OAAM,MAAK;GACV,KAAKD,IAAI,OAAO,gCAAgC,CAAC;GACjD,IAAI,KAAK,CAAC;EACZ,CAAC;EACH,OAAO;CACT;CAEA,eAAe,QAAgB,qBAAqB,MAAY;EAC9D,MAAM,MAAM,KAAKM,cAAc,IAAI,MAAM;EACzC,IAAI,QAAQ,KAAA,GACV;EAEF,KAAKA,cAAc,OAAO,MAAM;EAChC,MAAM,EAAC,OAAO,QAAO;EACrB,IAAI,OAAO;EAEX,IAAI,oBAAoB;GACtB,MAAM,WAAW,KAAK,IAAI,IAAI,MAAM,QAAQ;GAC5C,KAAKN,IAAI,OAAO,2BAA2B,OAAO,MAAM,SAAS,IAAI;GACrE,IAAI,WAAW,KAAKS,iBAAiB;IACnC,KAAKA,kBAAkB;IACvB,KAAKT,IAAI,OAAO,8BAA8B,SAAS,IAAI;GAC7D;EACF;CACF;CAGA,mCAA4C,YAAY;EACtD,IAAI;GACF,MAAM,KAAKY,gBAAgB;EAC7B,SAAS,GAAG;GACV,KAAKJ,eAAe,IAAI,GAAG,EAAC,QAAQ,mBAAkB,CAAC;GACvD,KAAKR,IAAI,OACP,uFAEA,CACF;GACA;EACF;EAEA,KAAKa,iBAAiB;CACxB;CAEA,MAAMD,kBAAiC;EACrC,MAAM,YAAY,MAAM,KAAKT,QAAQ,cAAc;EACnD,KAAKW,yBAAyB;EAC9B,IACE,UAAU,YAAY,KAAKC,kBAC3B,CAAC,KAAKR,YAAY,IAAI,UAAU,SAAS,GACzC;GACA,KAAKP,IAAI,OACP,6BAA6B,UAAU,UAAU,6BAC1B,IAAI,KAAK,UAAU,YAAY,EAAE,YAAY,KACpE;IACE,aAAa,UAAU;IACvB,MAAM,UAAU;IAChB,YAAY,UAAU;GACxB,CACF;GACA,KAAKO,YAAY,IAAI,UAAU,WAAW,SAAS;EACrD;CACF;CAEA,mBAAyB;EACvB,IAAI,KAAKD,cAAc,OAAO,GAAG;GAC/B,KAAKN,IAAI,OACP,6CAA6C,CAAC,GAAG,KAAKM,cAAc,KAAK,CAAC,GAC5E;GACA;EACF;EAEA,MAAM,2BAA2B,KAAKQ,wBAAwB;EAC9D,IAAI,6BAA6B,KAAA,GAC/B;EAGF,MAAM,eAAe,KAAKE,kBACxB,KAAK,IAAI,IAAI,KAAKP,iBAClB,wBACF;EACA,IAAI,aAAa,WAAW,GAC1B;EAGF,KAAKP,gBAAgB,gBAAgB,YAAY;EACjD,KAAK,MAAM,aAAa,KAAKK,YAAY,KAAK,GAC5C,IAAI,aAAa,cACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKQ,iBAAiB;CACxB;CAEA,kBAAkB,QAAgB,0BAA0C;EAC1E,IAAI,MAAM;EACV,KAAK,MAAM,CAAC,WAAW,oBAAoB,KAAKR,aAC9C,IACE,aAAa,4BACb,gBAAgB,gBAAgB,UAChC,YAAY,KAEZ,MAAM;EAGV,OAAO;CACT;CAEA,OAAsB;EACpB,cAAc,KAAKG,oBAAoB;EACvC,KAAK,MAAM,EAAC,SAAQ,KAAKJ,cAAc,OAAO,GAC5C,IAAI,OAAO;EAEb,KAAKH,QAAQ,QAAQ;EACrB,KAAKE,OAAO,KAAK,KAAKL,GAAG;EACzB,OAAO;CACT;CAEA,uBAA6B;EAC3B,iBAAiB,WAAW,cAAc;GACxC,aACE;GAEF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,wBAAwB,KAAKc;GACnC,IAAI,uBAAuB,gBAAgB,KAAA,GAAW;IACpD,KAAKd,IAAI,OACP,8DACF;IACA;GACF;GACA,IAAI,sBAAsB,gBAAgB,MACxC;GAEF,EAAE,QACA,KAAK,IACH,GACA,sBAAsB,eACpB,sBAAsB,WAC1B,CACF;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/litestream/commands.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAM3D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAiB5D,KAAK,kBAAkB,GAAG;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;gBAErC,SAAS,EAAE,MAAM,GAAG,SAAS;CAG1C;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,kBAAkB,EAAE,kBAAkB,GAAG,IAAI,iBA8B9C;AAkKD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAQd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAgBf;AAqDD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}
+{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAM3D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAe5D,KAAK,kBAAkB,GAAG;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;gBAErC,SAAS,EAAE,MAAM,GAAG,SAAS;CAG1C;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,kBAAkB,EAAE,kBAAkB,GAAG,IAAI,iBA4C9C;AAoKD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAQd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAgBf;AAqDD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}

package/out/zero-cache/src/services/litestream/commands.js

@@ -6,12 +6,12 @@ import { Database } from "../../../../zqlite/src/db.js";
 import { getSubscriptionState } from "../replicator/schema/replication-state.js";
 import { StatementRunner } from "../../db/statements.js";
 import { deleteLiteDB } from "../../db/delete-lite-db.js";
+import { assertDatabaseIntegrity } from "../../db/migration-lite.js";
 import { ChangeStreamerHttpClient } from "../change-streamer/change-streamer-http.js";
 import { resolver } from "@rocicorp/resolver";
 import { spawn } from "node:child_process";
 import { existsSync } from "node:fs";
 //#region ../zero-cache/src/services/litestream/commands.ts
-var MAX_RETRIES = 60;
 var RETRY_INTERVAL_MS = 3e3;
 var BackupNotFoundException = class extends Error {
 	static name = "BackupNotFoundException";
@@ -26,12 +26,14 @@ var BackupNotFoundException = class extends Error {
 *        retrieved from the replication-manager via the snapshot protocol.
 */
 async function restoreReplica(lc, config, replicaConstraints) {
-	for (let i = 0; i < MAX_RETRIES; i++) {
+	let consecutiveErrors = 0;
+	for (;;) {
 		try {
 			if (await tryRestore(lc, config, replicaConstraints)) return;
+			consecutiveErrors = 0;
 		} catch (e) {
-			if (i === 0) {
-				lc.warn?.(`initial restore attempt failed. retrying once`, e);
+			if (++consecutiveErrors <= 1) {
+				lc.warn?.(`restore attempt failed. retrying once`, e);
 				continue;
 			}
 			throw e;
@@ -40,7 +42,6 @@ async function restoreReplica(lc, config, replicaConstraints) {
 		lc.info?.(`replica not found. retrying in ${RETRY_INTERVAL_MS / 1e3} seconds`);
 		await sleep(RETRY_INTERVAL_MS);
 	}
-	throw new Error(`max attempts exceeded restoring replica`);
 }
 function getLitestream(mode, config, logLevelOverride, backupURLOverride) {
 	const { executable, executableV5, restoreUsingV5, backupURL, logLevel, configPath, endpoint, region, port = config.port + 2, checkpointThresholdMB, minCheckpointPageCount = checkpointThresholdMB * 250, maxCheckpointPageCount = minCheckpointPageCount * 10, incrementalBackupIntervalMinutes, snapshotBackupIntervalHours, multipartConcurrency, multipartSize } = config.litestream;
@@ -104,8 +105,10 @@ async function tryRestore(lc, config, replicaConstraints) {
 	return true;
 }
 function replicaIsValid(lc, replica, constraints) {
-	const db = new Database(lc, replica);
+	let db;
 	try {
+		db = new Database(lc, replica);
+		assertDatabaseIntegrity(lc, "restored replica", db);
 		const { replicaVersion, watermark } = getSubscriptionState(new StatementRunner(db));
 		if (replicaVersion !== constraints.replicaVersion) {
 			lc.warn?.(`Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`, constraints);
@@ -121,7 +124,7 @@ function replicaIsValid(lc, replica, constraints) {
 		lc.error?.("Error while validating restored replica", e);
 		return false;
 	} finally {
-		db.close();
+		db?.close();
 	}
 }
 function startReplicaBackupProcess(lc, config) {

package/out/zero-cache/src/services/litestream/commands.js.map

@@ -1 +1 @@
-{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\ntype ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * @param replicaConstraints The constraints of the restored backup when\n *        restoring for the change-streamer (replication-manager). For the\n *        view-syncer, this should be unspecified so that the constraints are\n *        retrieved from the replication-manager via the snapshot protocol.\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  for (let i = 0; i < MAX_RETRIES; i++) {\n    try {\n      if (await tryRestore(lc, config, replicaConstraints)) {\n        return;\n      }\n    } catch (e) {\n      if (i === 0) {\n        // A restore will fail if the `replicate` process creates a new\n        // snapshot (and compacts old files) at the same time. Snapshots are\n        // infrequent (e.g. once every 12 hours), and the scenario is\n        // recoverable with a retry.\n        lc.warn?.(`initial restore attempt failed. retrying once`, e);\n        continue;\n      }\n      // If it fails again on the retry, though, bail.\n      throw e;\n    }\n    if (replicaConstraints) {\n      // This can happen if the litestream URL is purposefully changed to\n      // force a resync.\n      throw new BackupNotFoundException(config.litestream.backupURL);\n    }\n    lc.info?.(\n      `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n    );\n    await sleep(RETRY_INTERVAL_MS);\n  }\n  throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupURL,\n    logLevel,\n    configPath,\n    endpoint,\n    region,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  const litestream =\n    // The v0.5.8+ litestream executable can restore from either the new LTX\n    // format or the legacy WAL format, allowing forwards-compatibility /\n    // rollback safety with zero-cache versions that backup to LTX.\n    (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n    },\n  };\n}\n\nasync function tryRestore(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (!replicaConstraints) {\n    // view-syncers fetch replica constraints from the replication-manager\n    // via the snapshot protocol.\n    snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n    replicaConstraints = snapshotStatus;\n  }\n\n  const {litestream, env} = getLitestream(\n    'restore',\n    config,\n    'debug', // Include all output from `litestream restore`, as it's minimal.\n    snapshotStatus?.backupURL,\n  );\n  const {restoreParallelism: parallelism} = config.litestream;\n  const proc = spawn(\n    litestream,\n    [\n      'restore',\n      '-if-db-not-exists',\n      '-if-replica-exists',\n      '-parallelism',\n      String(parallelism),\n      config.replica.file,\n    ],\n    {env, stdio: 'inherit', windowsHide: true},\n  );\n  const {promise, resolve, reject} = resolver();\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(`litestream killed with ${signal}`);\n    } else if (code !== 0) {\n      reject(`litestream exited with code ${code}`);\n    } else {\n      resolve();\n    }\n  });\n  await promise;\n  if (!existsSync(config.replica.file)) {\n    return false;\n  }\n  if (!replicaIsValid(lc, config.replica.file, replicaConstraints)) {\n    lc.info?.(`Deleting local replica and retrying restore`);\n    deleteLiteDB(config.replica.file);\n    return false;\n  }\n  return true;\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints,\n) {\n  const db = new Database(lc, replica);\n  try {\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== constraints.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n        constraints,\n      );\n      return false;\n    }\n    if (watermark < constraints.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n      constraints,\n    );\n    return true;\n  } catch (e) {\n    lc.error?.('Error while validating restored replica', e);\n    return false;\n  } finally {\n    db.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: ZeroConfig,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config);\n  lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n  return spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, 'snapshots'),\n    listBackupCreatedTimes(lc, config, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${config.litestream.backupURL}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: ZeroConfig,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const {litestream, env} = getLitestream('replicate', config);\n  const proc = spawn(litestream, [command, config.replica.file], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  let output: string;\n  try {\n    output = await promise;\n  } finally {\n    clearTimeout(timeout);\n  }\n\n  return parseBackupCreatedTimes(lc, command, output);\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // (i.e. BackupMonitor) returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,IAAM,cAAc;AACpB,IAAM,oBAAoB;AAO1B,IAAa,0BAAb,cAA6C,MAAM;CACjD,OAAgB,OAAO;CAEvB,YAAY,WAA+B;EACzC,MAAM,uBAAuB,WAAW;CAC1C;AACF;;;;;;;AAQA,eAAsB,eACpB,IACA,QACA,oBACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAAK;EACpC,IAAI;GACF,IAAI,MAAM,WAAW,IAAI,QAAQ,kBAAkB,GACjD;EAEJ,SAAS,GAAG;GACV,IAAI,MAAM,GAAG;IAKX,GAAG,OAAO,iDAAiD,CAAC;IAC5D;GACF;GAEA,MAAM;EACR;EACA,IAAI,oBAGF,MAAM,IAAI,wBAAwB,OAAO,WAAW,SAAS;EAE/D,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;EACA,MAAM,MAAM,iBAAiB;CAC/B;CACA,MAAM,IAAI,MAAM,yCAAyC;AAC3D;AAEA,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,QACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;CAQzE,OAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,iCAAiC;EAGlD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,SAAS;IAClE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,6BACF;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAe,WACb,IACA,QACA,oBACA;CACA,IAAI;CACJ,IAAI,CAAC,oBAAoB;EAGvB,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;EAC7D,GAAG,OAAO,yBAAyB,eAAe,WAAW;EAC7D,qBAAqB;CACvB;CAEA,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,SAClB;CACA,MAAM,EAAC,oBAAoB,gBAAe,OAAO;CACjD,MAAM,OAAO,MACX,YACA;EACE;EACA;EACA;EACA;EACA,OAAO,WAAW;EAClB,OAAO,QAAQ;CACjB,GACA;EAAC;EAAK,OAAO;EAAW,aAAa;CAAI,CAC3C;CACA,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;CAC5C,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;OACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;OAE5C,QAAQ;CAEZ,CAAC;CACD,MAAM;CACN,IAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GACjC,OAAO;CAET,IAAI,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,kBAAkB,GAAG;EAChE,GAAG,OAAO,6CAA6C;EACvD,aAAa,OAAO,QAAQ,IAAI;EAChC,OAAO;CACT;CACA,OAAO;AACT;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,MAAM,KAAK,IAAI,SAAS,IAAI,OAAO;CACnC,IAAI;EACF,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,mBAAmB,YAAY,gBAAgB;GACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;GACA,OAAO;EACT;EACA,IAAI,YAAY,YAAY,cAAc;GACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;GACA,OAAO;EACT;EACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,GAAG,MAAM;CACX;AACF;AAEA,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,GAAG,OAAO,iCAAiC,OAAO,WAAW,WAAW;CACxE,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EACtC;EACA,OAAO;EACP,aAAa;CACf,CAAC;AACH;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,WAAW,GAC9C,uBAAuB,IAAI,QAAQ,KAAK,CAC1C,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WAAW,WAC9D;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,SACiB;CACjB,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,OAAO,QAAQ,IAAI,GAAG;EAC7D;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;CACJ,IAAI;EACF,SAAS,MAAM;CACjB,UAAU;EACR,aAAa,OAAO;CACtB;CAEA,OAAO,wBAAwB,IAAI,SAAS,MAAM;AACpD;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,4BACP,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,iBAAiB,MAAM;CACvB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}
+{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {assertDatabaseIntegrity} from '../../db/migration-lite.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\nconst RETRY_INTERVAL_MS = 3000;\n\ntype ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * @param replicaConstraints The constraints of the restored backup when\n *        restoring for the change-streamer (replication-manager). For the\n *        view-syncer, this should be unspecified so that the constraints are\n *        retrieved from the replication-manager via the snapshot protocol.\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  // View-syncers (no replicaConstraints) wait indefinitely for the\n  // replication-manager to publish a restorable backup. On a fresh stack the\n  // first backup is not durable until the initial sync completes and litestream\n  // uploads the initial snapshot, which can take many minutes for a large\n  // replica. The platform's startup probe budget (which scales with replica\n  // size) is the backstop, so restoreReplica must not impose its own shorter\n  // cap and self-terminate while the backup is still being produced.\n  //\n  // `consecutiveErrors` distinguishes a transient restore *error* (e.g. the\n  // `replicate` process compacting a snapshot mid-restore) — which is retried\n  // once — from a \"backup not found\" result, which is expected while waiting.\n  let consecutiveErrors = 0;\n  for (;;) {\n    try {\n      if (await tryRestore(lc, config, replicaConstraints)) {\n        return;\n      }\n      consecutiveErrors = 0;\n    } catch (e) {\n      if (++consecutiveErrors <= 1) {\n        // A restore will fail if the `replicate` process creates a new\n        // snapshot (and compacts old files) at the same time. Snapshots are\n        // infrequent (e.g. once every 12 hours), and the scenario is\n        // recoverable with a retry.\n        lc.warn?.(`restore attempt failed. retrying once`, e);\n        continue;\n      }\n      // If it fails again on the retry, though, bail.\n      throw e;\n    }\n    if (replicaConstraints) {\n      // The replication-manager restores against explicit constraints, so a\n      // missing backup is fatal (e.g. the litestream URL was purposefully\n      // changed to force a resync). Only the view-syncer (no constraints)\n      // waits for a backup to appear.\n      throw new BackupNotFoundException(config.litestream.backupURL);\n    }\n    lc.info?.(\n      `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n    );\n    await sleep(RETRY_INTERVAL_MS);\n  }\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupURL,\n    logLevel,\n    configPath,\n    endpoint,\n    region,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  const litestream =\n    // The v0.5.8+ litestream executable can restore from either the new LTX\n    // format or the legacy WAL format, allowing forwards-compatibility /\n    // rollback safety with zero-cache versions that backup to LTX.\n    (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n    },\n  };\n}\n\nasync function tryRestore(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (!replicaConstraints) {\n    // view-syncers fetch replica constraints from the replication-manager\n    // via the snapshot protocol.\n    snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n    replicaConstraints = snapshotStatus;\n  }\n\n  const {litestream, env} = getLitestream(\n    'restore',\n    config,\n    'debug', // Include all output from `litestream restore`, as it's minimal.\n    snapshotStatus?.backupURL,\n  );\n  const {restoreParallelism: parallelism} = config.litestream;\n  const proc = spawn(\n    litestream,\n    [\n      'restore',\n      '-if-db-not-exists',\n      '-if-replica-exists',\n      '-parallelism',\n      String(parallelism),\n      config.replica.file,\n    ],\n    {env, stdio: 'inherit', windowsHide: true},\n  );\n  const {promise, resolve, reject} = resolver();\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(`litestream killed with ${signal}`);\n    } else if (code !== 0) {\n      reject(`litestream exited with code ${code}`);\n    } else {\n      resolve();\n    }\n  });\n  await promise;\n  if (!existsSync(config.replica.file)) {\n    return false;\n  }\n  if (!replicaIsValid(lc, config.replica.file, replicaConstraints)) {\n    lc.info?.(`Deleting local replica and retrying restore`);\n    deleteLiteDB(config.replica.file);\n    return false;\n  }\n  return true;\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints,\n) {\n  let db: Database | undefined;\n  try {\n    db = new Database(lc, replica);\n    assertDatabaseIntegrity(lc, 'restored replica', db);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== constraints.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n        constraints,\n      );\n      return false;\n    }\n    if (watermark < constraints.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n      constraints,\n    );\n    return true;\n  } catch (e) {\n    lc.error?.('Error while validating restored replica', e);\n    return false;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: ZeroConfig,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config);\n  lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n  return spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, 'snapshots'),\n    listBackupCreatedTimes(lc, config, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${config.litestream.backupURL}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: ZeroConfig,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const {litestream, env} = getLitestream('replicate', config);\n  const proc = spawn(litestream, [command, config.replica.file], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  let output: string;\n  try {\n    output = await promise;\n  } finally {\n    clearTimeout(timeout);\n  }\n\n  return parseBackupCreatedTimes(lc, command, output);\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // backup monitor returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;;AAsBA,IAAM,oBAAoB;AAO1B,IAAa,0BAAb,cAA6C,MAAM;CACjD,OAAgB,OAAO;CAEvB,YAAY,WAA+B;EACzC,MAAM,uBAAuB,WAAW;CAC1C;AACF;;;;;;;AAQA,eAAsB,eACpB,IACA,QACA,oBACA;CAYA,IAAI,oBAAoB;CACxB,SAAS;EACP,IAAI;GACF,IAAI,MAAM,WAAW,IAAI,QAAQ,kBAAkB,GACjD;GAEF,oBAAoB;EACtB,SAAS,GAAG;GACV,IAAI,EAAE,qBAAqB,GAAG;IAK5B,GAAG,OAAO,yCAAyC,CAAC;IACpD;GACF;GAEA,MAAM;EACR;EACA,IAAI,oBAKF,MAAM,IAAI,wBAAwB,OAAO,WAAW,SAAS;EAE/D,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;EACA,MAAM,MAAM,iBAAiB;CAC/B;AACF;AAEA,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,QACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;CAQzE,OAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,iCAAiC;EAGlD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,SAAS;IAClE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,6BACF;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAe,WACb,IACA,QACA,oBACA;CACA,IAAI;CACJ,IAAI,CAAC,oBAAoB;EAGvB,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;EAC7D,GAAG,OAAO,yBAAyB,eAAe,WAAW;EAC7D,qBAAqB;CACvB;CAEA,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,SAClB;CACA,MAAM,EAAC,oBAAoB,gBAAe,OAAO;CACjD,MAAM,OAAO,MACX,YACA;EACE;EACA;EACA;EACA;EACA,OAAO,WAAW;EAClB,OAAO,QAAQ;CACjB,GACA;EAAC;EAAK,OAAO;EAAW,aAAa;CAAI,CAC3C;CACA,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;CAC5C,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;OACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;OAE5C,QAAQ;CAEZ,CAAC;CACD,MAAM;CACN,IAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GACjC,OAAO;CAET,IAAI,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,kBAAkB,GAAG;EAChE,GAAG,OAAO,6CAA6C;EACvD,aAAa,OAAO,QAAQ,IAAI;EAChC,OAAO;CACT;CACA,OAAO;AACT;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,wBAAwB,IAAI,oBAAoB,EAAE;EAClD,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,mBAAmB,YAAY,gBAAgB;GACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;GACA,OAAO;EACT;EACA,IAAI,YAAY,YAAY,cAAc;GACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;GACA,OAAO;EACT;EACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,GAAG,OAAO,iCAAiC,OAAO,WAAW,WAAW;CACxE,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EACtC;EACA,OAAO;EACP,aAAa;CACf,CAAC;AACH;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,WAAW,GAC9C,uBAAuB,IAAI,QAAQ,KAAK,CAC1C,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WAAW,WAC9D;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,SACiB;CACjB,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,OAAO,QAAQ,IAAI,GAAG;EAC7D;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;CACJ,IAAI;EACF,SAAS,MAAM;CACjB,UAAU;EACR,aAAa,OAAO;CACtB;CAEA,OAAO,wBAAwB,IAAI,SAAS,MAAM;AACpD;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,4BACP,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,iBAAiB,MAAM;CACvB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.d.ts

@@ -0,0 +1,44 @@
+import type { LogContext, LogLevel } from '@rocicorp/logger';
+type LoadableDatabase = {
+    loadExtension(path: string): unknown;
+    close(): void;
+};
+type QueryDatabase = {
+    prepare(sql: string): {
+        get<T>(): T;
+    };
+    close(): void;
+};
+export type VfsBackupWatermark = {
+    watermark: string;
+    writeTimeMs: number | null;
+    txid: string;
+    lagSeconds: number;
+    observedAtMs: number;
+};
+export type VfsBackupWatermarkReaderOptions = {
+    readonly replicaURL: string;
+    readonly extensionPath: string;
+    readonly logLevel?: LogLevel | undefined;
+    readonly logFile?: string | undefined;
+    readonly databaseURI?: string | undefined;
+    readonly env?: NodeJS.ProcessEnv | undefined;
+};
+type VfsBackupWatermarkReaderDeps = {
+    readonly loaderFactory?: (() => LoadableDatabase) | undefined;
+    readonly databaseFactory?: ((lc: LogContext, uri: string) => QueryDatabase) | undefined;
+};
+export declare function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region, }: {
+    readonly backupURL: string;
+    readonly endpoint?: string | undefined;
+    readonly region?: string | undefined;
+}): string;
+export declare class VfsBackupWatermarkReader implements Disposable {
+    #private;
+    constructor(lc: LogContext, options: VfsBackupWatermarkReaderOptions, deps?: VfsBackupWatermarkReaderDeps);
+    readWatermark(): VfsBackupWatermark;
+    close(): void;
+    [Symbol.dispose](): void;
+}
+export {};
+//# sourceMappingURL=vfs-watermark-reader.d.ts.map

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-reader.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAE,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAQ3D,KAAK,gBAAgB,GAAG;IACtB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IACrC,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AAEF,KAAK,aAAa,GAAG;IACnB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG;QACpB,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;KACb,CAAC;IACF,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG;IAC5C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,KAAK,4BAA4B,GAAG;IAClC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,MAAM,gBAAgB,CAAC,GAAG,SAAS,CAAC;IAC9D,QAAQ,CAAC,eAAe,CAAC,EACrB,CAAC,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,KAAK,aAAa,CAAC,GAChD,SAAS,CAAC;CACf,CAAC;AASF,wBAAgB,4BAA4B,CAAC,EAC3C,SAAS,EACT,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,GAAG,MAAM,CAiBT;AAED,qBAAa,wBAAyB,YAAW,UAAU;;gBAQvD,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,+BAA+B,EACxC,IAAI,GAAE,4BAAiC;IAiCzC,aAAa,IAAI,kBAAkB;IAQnC,KAAK,IAAI,IAAI;IAKb,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAGzB"}

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.js

@@ -0,0 +1,65 @@
+import { parse, valita_exports } from "../../../../shared/src/valita.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import SQLite3Database from "@rocicorp/zero-sqlite3";
+//#region ../zero-cache/src/services/litestream/vfs-watermark-reader.ts
+var DEFAULT_VFS_DATABASE_URI = "file:zero-backup.db?vfs=litestream&mode=ro";
+var DEFAULT_VFS_LOG_LEVEL = "info";
+var backupWatermarkRowSchema = valita_exports.object({
+	watermark: valita_exports.string(),
+	writeTimeMs: valita_exports.number().nullable(),
+	txid: valita_exports.string(),
+	lagSeconds: valita_exports.number()
+});
+function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region }) {
+	if (endpoint === void 0 && region === void 0) return backupURL;
+	const url = new URL(backupURL);
+	if (url.protocol !== "s3:") return backupURL;
+	if (endpoint !== void 0 && !url.searchParams.has("endpoint")) url.searchParams.set("endpoint", endpoint);
+	if (region !== void 0 && !url.searchParams.has("region")) url.searchParams.set("region", region);
+	return url.toString();
+}
+var VfsBackupWatermarkReader = class {
+	#loader;
+	#db;
+	#stmt;
+	constructor(lc, options, deps = {}) {
+		const env = options.env ?? process.env;
+		env["LITESTREAM_REPLICA_URL"] = options.replicaURL;
+		env["LITESTREAM_LOG_LEVEL"] = (options.logLevel ?? DEFAULT_VFS_LOG_LEVEL).toUpperCase();
+		setOptionalEnv(env, "LITESTREAM_LOG_FILE", options.logFile);
+		const loaderFactory = deps.loaderFactory ?? (() => new SQLite3Database(":memory:"));
+		const databaseFactory = deps.databaseFactory ?? ((lc, uri) => new Database(lc, uri, { readonly: true }));
+		this.#loader = loaderFactory();
+		this.#loader.loadExtension(options.extensionPath);
+		this.#db = databaseFactory(lc.withContext("component", "vfs-backup-watermark-reader"), options.databaseURI ?? DEFAULT_VFS_DATABASE_URI);
+		this.#stmt = this.#db.prepare(`
+      SELECT
+        stateVersion AS watermark,
+        writeTimeMs,
+        litestream_txid() AS txid,
+        litestream_lag() AS lagSeconds
+      FROM "_zero.replicationState"
+    `);
+	}
+	readWatermark() {
+		return {
+			...parse(this.#stmt.get(), backupWatermarkRowSchema),
+			observedAtMs: Date.now()
+		};
+	}
+	close() {
+		this.#db.close();
+		this.#loader.close();
+	}
+	[Symbol.dispose]() {
+		this.close();
+	}
+};
+function setOptionalEnv(env, name, value) {
+	if (value === void 0) delete env[name];
+	else env[name] = value;
+}
+//#endregion
+export { VfsBackupWatermarkReader, buildLitestreamVfsReplicaURL };
+
+//# sourceMappingURL=vfs-watermark-reader.js.map

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-reader.js","names":["#loader","#db","#stmt"],"sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-reader.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport Sqlite3Database from '@rocicorp/zero-sqlite3';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\n\nconst DEFAULT_VFS_DATABASE_URI = 'file:zero-backup.db?vfs=litestream&mode=ro';\nconst DEFAULT_VFS_LOG_LEVEL = 'info';\n\ntype LoadableDatabase = {\n  loadExtension(path: string): unknown;\n  close(): void;\n};\n\ntype QueryDatabase = {\n  prepare(sql: string): {\n    get<T>(): T;\n  };\n  close(): void;\n};\n\nexport type VfsBackupWatermark = {\n  watermark: string;\n  writeTimeMs: number | null;\n  txid: string;\n  lagSeconds: number;\n  observedAtMs: number;\n};\n\nexport type VfsBackupWatermarkReaderOptions = {\n  readonly replicaURL: string;\n  readonly extensionPath: string;\n  readonly logLevel?: LogLevel | undefined;\n  readonly logFile?: string | undefined;\n  readonly databaseURI?: string | undefined;\n  readonly env?: NodeJS.ProcessEnv | undefined;\n};\n\ntype VfsBackupWatermarkReaderDeps = {\n  readonly loaderFactory?: (() => LoadableDatabase) | undefined;\n  readonly databaseFactory?:\n    | ((lc: LogContext, uri: string) => QueryDatabase)\n    | undefined;\n};\n\nconst backupWatermarkRowSchema = v.object({\n  watermark: v.string(),\n  writeTimeMs: v.number().nullable(),\n  txid: v.string(),\n  lagSeconds: v.number(),\n});\n\nexport function buildLitestreamVfsReplicaURL({\n  backupURL,\n  endpoint,\n  region,\n}: {\n  readonly backupURL: string;\n  readonly endpoint?: string | undefined;\n  readonly region?: string | undefined;\n}): string {\n  if (endpoint === undefined && region === undefined) {\n    return backupURL;\n  }\n\n  const url = new URL(backupURL);\n  if (url.protocol !== 's3:') {\n    return backupURL;\n  }\n\n  if (endpoint !== undefined && !url.searchParams.has('endpoint')) {\n    url.searchParams.set('endpoint', endpoint);\n  }\n  if (region !== undefined && !url.searchParams.has('region')) {\n    url.searchParams.set('region', region);\n  }\n  return url.toString();\n}\n\nexport class VfsBackupWatermarkReader implements Disposable {\n  readonly #loader: LoadableDatabase;\n  readonly #db: QueryDatabase;\n  readonly #stmt: {\n    get<T>(): T;\n  };\n\n  constructor(\n    lc: LogContext,\n    options: VfsBackupWatermarkReaderOptions,\n    deps: VfsBackupWatermarkReaderDeps = {},\n  ) {\n    const env = options.env ?? process.env;\n    env['LITESTREAM_REPLICA_URL'] = options.replicaURL;\n    env['LITESTREAM_LOG_LEVEL'] = (\n      options.logLevel ?? DEFAULT_VFS_LOG_LEVEL\n    ).toUpperCase();\n    setOptionalEnv(env, 'LITESTREAM_LOG_FILE', options.logFile);\n\n    const loaderFactory =\n      deps.loaderFactory ?? (() => new Sqlite3Database(':memory:'));\n    const databaseFactory =\n      deps.databaseFactory ??\n      ((lc: LogContext, uri: string) =>\n        new Database(lc, uri, {readonly: true}));\n\n    this.#loader = loaderFactory();\n    this.#loader.loadExtension(options.extensionPath);\n\n    this.#db = databaseFactory(\n      lc.withContext('component', 'vfs-backup-watermark-reader'),\n      options.databaseURI ?? DEFAULT_VFS_DATABASE_URI,\n    );\n    this.#stmt = this.#db.prepare(/*sql*/ `\n      SELECT\n        stateVersion AS watermark,\n        writeTimeMs,\n        litestream_txid() AS txid,\n        litestream_lag() AS lagSeconds\n      FROM \"_zero.replicationState\"\n    `);\n  }\n\n  readWatermark(): VfsBackupWatermark {\n    const row = v.parse(this.#stmt.get(), backupWatermarkRowSchema);\n    return {\n      ...row,\n      observedAtMs: Date.now(),\n    };\n  }\n\n  close(): void {\n    this.#db.close();\n    this.#loader.close();\n  }\n\n  [Symbol.dispose](): void {\n    this.close();\n  }\n}\n\nfunction setOptionalEnv(\n  env: NodeJS.ProcessEnv,\n  name: string,\n  value: string | undefined,\n) {\n  if (value === undefined) {\n    delete env[name];\n  } else {\n    env[name] = value;\n  }\n}\n"],"mappings":";;;;AAKA,IAAM,2BAA2B;AACjC,IAAM,wBAAwB;AAsC9B,IAAM,2BAA2B,eAAE,OAAO;CACxC,WAAW,eAAE,OAAO;CACpB,aAAa,eAAE,OAAO,EAAE,SAAS;CACjC,MAAM,eAAE,OAAO;CACf,YAAY,eAAE,OAAO;AACvB,CAAC;AAED,SAAgB,6BAA6B,EAC3C,WACA,UACA,UAKS;CACT,IAAI,aAAa,KAAA,KAAa,WAAW,KAAA,GACvC,OAAO;CAGT,MAAM,MAAM,IAAI,IAAI,SAAS;CAC7B,IAAI,IAAI,aAAa,OACnB,OAAO;CAGT,IAAI,aAAa,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,UAAU,GAC5D,IAAI,aAAa,IAAI,YAAY,QAAQ;CAE3C,IAAI,WAAW,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,QAAQ,GACxD,IAAI,aAAa,IAAI,UAAU,MAAM;CAEvC,OAAO,IAAI,SAAS;AACtB;AAEA,IAAa,2BAAb,MAA4D;CAC1D;CACA;CACA;CAIA,YACE,IACA,SACA,OAAqC,CAAC,GACtC;EACA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EACnC,IAAI,4BAA4B,QAAQ;EACxC,IAAI,2BACF,QAAQ,YAAY,uBACpB,YAAY;EACd,eAAe,KAAK,uBAAuB,QAAQ,OAAO;EAE1D,MAAM,gBACJ,KAAK,wBAAwB,IAAI,gBAAgB,UAAU;EAC7D,MAAM,kBACJ,KAAK,qBACH,IAAgB,QAChB,IAAI,SAAS,IAAI,KAAK,EAAC,UAAU,KAAI,CAAC;EAE1C,KAAKA,UAAU,cAAc;EAC7B,KAAKA,QAAQ,cAAc,QAAQ,aAAa;EAEhD,KAAKC,MAAM,gBACT,GAAG,YAAY,aAAa,6BAA6B,GACzD,QAAQ,eAAe,wBACzB;EACA,KAAKC,QAAQ,KAAKD,IAAI,QAAgB;;;;;;;KAOrC;CACH;CAEA,gBAAoC;EAElC,OAAO;GACL,GAFU,MAAQ,KAAKC,MAAM,IAAI,GAAG,wBAEjC;GACH,cAAc,KAAK,IAAI;EACzB;CACF;CAEA,QAAc;EACZ,KAAKD,IAAI,MAAM;EACf,KAAKD,QAAQ,MAAM;CACrB;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,MAAM;CACb;AACF;AAEA,SAAS,eACP,KACA,MACA,OACA;CACA,IAAI,UAAU,KAAA,GACZ,OAAO,IAAI;MAEX,IAAI,QAAQ;AAEhB"}

package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.d.ts

@@ -0,0 +1,11 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Worker } from '../../types/processes.ts';
+import type { VfsBackupWatermark } from './vfs-watermark-reader.ts';
+export type VfsBackupWatermarkWorkerFactory = () => Worker;
+export declare class VfsBackupWatermarkWorkerSource {
+    #private;
+    constructor(lc: LogContext, workerFactory: VfsBackupWatermarkWorkerFactory, timeoutMs: number);
+    readWatermark(): Promise<VfsBackupWatermark>;
+    close(): void;
+}
+//# sourceMappingURL=vfs-watermark-worker-source.d.ts.map

package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-worker-source.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-worker-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,2BAA2B,CAAC;AAMlE,MAAM,MAAM,+BAA+B,GAAG,MAAM,MAAM,CAAC;AAE3D,qBAAa,8BAA8B;;gBASvC,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,+BAA+B,EAC9C,SAAS,EAAE,MAAM;IAUb,aAAa,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAclD,KAAK,IAAI,IAAI;CA4Dd"}

package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.js

@@ -0,0 +1,74 @@
+import { BackupWatermarkTimeoutError, requestVfsBackupWatermark } from "./vfs-watermark-worker.js";
+//#region ../zero-cache/src/services/litestream/vfs-watermark-worker-source.ts
+var VfsBackupWatermarkWorkerSource = class {
+	#lc;
+	#workerFactory;
+	#timeoutMs;
+	#worker;
+	#ready;
+	constructor(lc, workerFactory, timeoutMs) {
+		this.#lc = lc.withContext("component", "vfs-backup-watermark-worker-source");
+		this.#workerFactory = workerFactory;
+		this.#timeoutMs = timeoutMs;
+	}
+	async readWatermark() {
+		const worker = await this.#getWorker();
+		try {
+			return await requestVfsBackupWatermark(worker, this.#timeoutMs, { killOnTimeout: true });
+		} catch (e) {
+			if (e instanceof BackupWatermarkTimeoutError) this.#clearWorker(worker);
+			throw e;
+		}
+	}
+	close() {
+		this.#worker?.kill("SIGTERM");
+		this.#worker = void 0;
+		this.#ready = void 0;
+	}
+	#getWorker() {
+		if (this.#ready) return this.#ready;
+		const worker = this.#workerFactory();
+		this.#worker = worker;
+		this.#ready = new Promise((resolve, reject) => {
+			const cleanup = () => {
+				worker.off("message", onMessage);
+				worker.off("error", onError);
+				worker.off("close", onClose);
+			};
+			const fail = (e) => {
+				cleanup();
+				this.#clearWorker(worker);
+				reject(e);
+			};
+			const onMessage = (data) => {
+				if (!isReadyMessage(data)) return;
+				cleanup();
+				resolve(worker);
+			};
+			const onError = (e) => fail(e);
+			const onClose = (code, signal) => fail(/* @__PURE__ */ new Error(`backup watermark reader worker exited before ready (code=${String(code)}, signal=${String(signal)})`));
+			worker.on("message", onMessage);
+			worker.once("error", onError);
+			worker.once("close", onClose);
+		});
+		worker.once("close", () => this.#clearWorker(worker));
+		worker.once("error", (e) => {
+			this.#lc.warn?.(`backup watermark reader worker failed`, e);
+			this.#clearWorker(worker);
+		});
+		return this.#ready;
+	}
+	#clearWorker(worker) {
+		if (this.#worker === worker) {
+			this.#worker = void 0;
+			this.#ready = void 0;
+		}
+	}
+};
+function isReadyMessage(data) {
+	return Array.isArray(data) && data.length === 2 && data[0] === "ready" && typeof data[1] === "object" && data[1] !== null && "ready" in data[1] && data[1].ready === true;
+}
+//#endregion
+export { VfsBackupWatermarkWorkerSource };
+
+//# sourceMappingURL=vfs-watermark-worker-source.js.map

package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-worker-source.js","names":["#lc","#workerFactory","#timeoutMs","#getWorker","#clearWorker","#worker","#ready"],"sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-worker-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Worker} from '../../types/processes.ts';\nimport type {VfsBackupWatermark} from './vfs-watermark-reader.ts';\nimport {\n  BackupWatermarkTimeoutError,\n  requestVfsBackupWatermark,\n} from './vfs-watermark-worker.ts';\n\nexport type VfsBackupWatermarkWorkerFactory = () => Worker;\n\nexport class VfsBackupWatermarkWorkerSource {\n  readonly #lc: LogContext;\n  readonly #workerFactory: VfsBackupWatermarkWorkerFactory;\n  readonly #timeoutMs: number;\n\n  #worker: Worker | undefined;\n  #ready: Promise<Worker> | undefined;\n\n  constructor(\n    lc: LogContext,\n    workerFactory: VfsBackupWatermarkWorkerFactory,\n    timeoutMs: number,\n  ) {\n    this.#lc = lc.withContext(\n      'component',\n      'vfs-backup-watermark-worker-source',\n    );\n    this.#workerFactory = workerFactory;\n    this.#timeoutMs = timeoutMs;\n  }\n\n  async readWatermark(): Promise<VfsBackupWatermark> {\n    const worker = await this.#getWorker();\n    try {\n      return await requestVfsBackupWatermark(worker, this.#timeoutMs, {\n        killOnTimeout: true,\n      });\n    } catch (e) {\n      if (e instanceof BackupWatermarkTimeoutError) {\n        this.#clearWorker(worker);\n      }\n      throw e;\n    }\n  }\n\n  close(): void {\n    this.#worker?.kill('SIGTERM');\n    this.#worker = undefined;\n    this.#ready = undefined;\n  }\n\n  #getWorker(): Promise<Worker> {\n    if (this.#ready) {\n      return this.#ready;\n    }\n\n    const worker = this.#workerFactory();\n    this.#worker = worker;\n    this.#ready = new Promise((resolve, reject) => {\n      const cleanup = () => {\n        worker.off('message', onMessage);\n        worker.off('error', onError);\n        worker.off('close', onClose);\n      };\n      const fail = (e: unknown) => {\n        cleanup();\n        this.#clearWorker(worker);\n        reject(e);\n      };\n      const onMessage = (data: unknown) => {\n        if (!isReadyMessage(data)) {\n          return;\n        }\n        cleanup();\n        resolve(worker);\n      };\n      const onError = (e: unknown) => fail(e);\n      const onClose = (code: unknown, signal: unknown) =>\n        fail(\n          new Error(\n            `backup watermark reader worker exited before ready ` +\n              `(code=${String(code)}, signal=${String(signal)})`,\n          ),\n        );\n\n      worker.on('message', onMessage);\n      worker.once('error', onError);\n      worker.once('close', onClose);\n    });\n\n    worker.once('close', () => this.#clearWorker(worker));\n    worker.once('error', e => {\n      this.#lc.warn?.(`backup watermark reader worker failed`, e);\n      this.#clearWorker(worker);\n    });\n\n    return this.#ready;\n  }\n\n  #clearWorker(worker: Worker): void {\n    if (this.#worker === worker) {\n      this.#worker = undefined;\n      this.#ready = undefined;\n    }\n  }\n}\n\nfunction isReadyMessage(data: unknown): boolean {\n  return (\n    Array.isArray(data) &&\n    data.length === 2 &&\n    data[0] === 'ready' &&\n    typeof data[1] === 'object' &&\n    data[1] !== null &&\n    'ready' in data[1] &&\n    data[1].ready === true\n  );\n}\n"],"mappings":";;AAUA,IAAa,iCAAb,MAA4C;CAC1C;CACA;CACA;CAEA;CACA;CAEA,YACE,IACA,eACA,WACA;EACA,KAAKA,MAAM,GAAG,YACZ,aACA,oCACF;EACA,KAAKC,iBAAiB;EACtB,KAAKC,aAAa;CACpB;CAEA,MAAM,gBAA6C;EACjD,MAAM,SAAS,MAAM,KAAKC,WAAW;EACrC,IAAI;GACF,OAAO,MAAM,0BAA0B,QAAQ,KAAKD,YAAY,EAC9D,eAAe,KACjB,CAAC;EACH,SAAS,GAAG;GACV,IAAI,aAAa,6BACf,KAAKE,aAAa,MAAM;GAE1B,MAAM;EACR;CACF;CAEA,QAAc;EACZ,KAAKC,SAAS,KAAK,SAAS;EAC5B,KAAKA,UAAU,KAAA;EACf,KAAKC,SAAS,KAAA;CAChB;CAEA,aAA8B;EAC5B,IAAI,KAAKA,QACP,OAAO,KAAKA;EAGd,MAAM,SAAS,KAAKL,eAAe;EACnC,KAAKI,UAAU;EACf,KAAKC,SAAS,IAAI,SAAS,SAAS,WAAW;GAC7C,MAAM,gBAAgB;IACpB,OAAO,IAAI,WAAW,SAAS;IAC/B,OAAO,IAAI,SAAS,OAAO;IAC3B,OAAO,IAAI,SAAS,OAAO;GAC7B;GACA,MAAM,QAAQ,MAAe;IAC3B,QAAQ;IACR,KAAKF,aAAa,MAAM;IACxB,OAAO,CAAC;GACV;GACA,MAAM,aAAa,SAAkB;IACnC,IAAI,CAAC,eAAe,IAAI,GACtB;IAEF,QAAQ;IACR,QAAQ,MAAM;GAChB;GACA,MAAM,WAAW,MAAe,KAAK,CAAC;GACtC,MAAM,WAAW,MAAe,WAC9B,qBACE,IAAI,MACF,4DACW,OAAO,IAAI,EAAE,WAAW,OAAO,MAAM,EAAE,EACpD,CACF;GAEF,OAAO,GAAG,WAAW,SAAS;GAC9B,OAAO,KAAK,SAAS,OAAO;GAC5B,OAAO,KAAK,SAAS,OAAO;EAC9B,CAAC;EAED,OAAO,KAAK,eAAe,KAAKA,aAAa,MAAM,CAAC;EACpD,OAAO,KAAK,UAAS,MAAK;GACxB,KAAKJ,IAAI,OAAO,yCAAyC,CAAC;GAC1D,KAAKI,aAAa,MAAM;EAC1B,CAAC;EAED,OAAO,KAAKE;CACd;CAEA,aAAa,QAAsB;EACjC,IAAI,KAAKD,YAAY,QAAQ;GAC3B,KAAKA,UAAU,KAAA;GACf,KAAKC,SAAS,KAAA;EAChB;CACF;AACF;AAEA,SAAS,eAAe,MAAwB;CAC9C,OACE,MAAM,QAAQ,IAAI,KAClB,KAAK,WAAW,KAChB,KAAK,OAAO,WACZ,OAAO,KAAK,OAAO,YACnB,KAAK,OAAO,QACZ,WAAW,KAAK,MAChB,KAAK,GAAG,UAAU;AAEtB"}

package/out/zero-cache/src/services/litestream/vfs-watermark-worker.d.ts

@@ -0,0 +1,42 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Message, Worker } from '../../types/processes.ts';
+import type { Service } from '../service.ts';
+import type { VfsBackupWatermark } from './vfs-watermark-reader.ts';
+export type BackupWatermarkRequest = [
+    'backupWatermarkRequest',
+    {
+        readonly requestID: string;
+    }
+];
+export type BackupWatermarkResponse = [
+    'backupWatermarkResponse',
+    {
+        readonly requestID: string;
+        readonly result?: VfsBackupWatermark | undefined;
+        readonly error?: {
+            readonly name: string;
+            readonly message: string;
+            readonly stack?: string | undefined;
+        } | undefined;
+    }
+];
+export interface BackupWatermarkSource {
+    readWatermark(): VfsBackupWatermark;
+    close?(): void;
+}
+export type BackupWatermarkSourceFactory = () => BackupWatermarkSource;
+export declare class BackupWatermarkTimeoutError extends Error {
+}
+export declare function requestVfsBackupWatermark(worker: Worker, timeoutMs: number, options?: {
+    readonly killOnTimeout?: boolean | undefined;
+    readonly killSignal?: NodeJS.Signals | undefined;
+}): Promise<VfsBackupWatermark>;
+export declare class VfsBackupWatermarkWorkerService implements Service {
+    #private;
+    readonly id = "vfs-backup-watermark-worker";
+    constructor(lc: LogContext, parent: Worker | null, source: BackupWatermarkSource | BackupWatermarkSourceFactory, pollIntervalMs: number);
+    run(): Promise<void>;
+    stop(): Promise<void>;
+}
+export type BackupWatermarkMessage = BackupWatermarkRequest | BackupWatermarkResponse | Message<unknown>;
+//# sourceMappingURL=vfs-watermark-worker.d.ts.map

package/out/zero-cache/src/services/litestream/vfs-watermark-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-worker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-worker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,OAAO,EAAE,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAE9D,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,2BAA2B,CAAC;AAElE,MAAM,MAAM,sBAAsB,GAAG;IACnC,wBAAwB;IACxB;QACE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B;CACF,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,yBAAyB;IACzB;QACE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC;QACjD,QAAQ,CAAC,KAAK,CAAC,EACX;YACE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;YACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;YACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;SACrC,GACD,SAAS,CAAC;KACf;CACF,CAAC;AAEF,MAAM,WAAW,qBAAqB;IACpC,aAAa,IAAI,kBAAkB,CAAC;IACpC,KAAK,CAAC,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,MAAM,4BAA4B,GAAG,MAAM,qBAAqB,CAAC;AAIvE,qBAAa,2BAA4B,SAAQ,KAAK;CAAG;AAEzD,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC7C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,OAAO,GAAG,SAAS,CAAC;CAC7C,GACL,OAAO,CAAC,kBAAkB,CAAC,CAkD7B;AAED,qBAAa,+BAAgC,YAAW,OAAO;;IAC7D,QAAQ,CAAC,EAAE,iCAAiC;gBAS1C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,MAAM,EAAE,qBAAqB,GAAG,4BAA4B,EAC5D,cAAc,EAAE,MAAM;IAQlB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IA+B1B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CA+BtB;AA6BD,MAAM,MAAM,sBAAsB,GAC9B,sBAAsB,GACtB,uBAAuB,GACvB,OAAO,CAAC,OAAO,CAAC,CAAC"}