@rocicorp/zero 1.9.1-canary.1 → 1.10.0-canary.1

package/out/replicache/src/lazy.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"lazy.d.ts","sourceRoot":"","sources":["../../../../replicache/src/lazy.ts"],"names":[],"mappings":"AAAA,wBAAgB,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,MAAM,CAAC,CAQjD"}

package/out/replicache/src/lazy.js

@@ -1,12 +0,0 @@
-//#region ../replicache/src/lazy.ts
-function lazy(factory) {
-	let value;
-	return () => {
-		if (value === void 0) value = factory();
-		return value;
-	};
-}
-//#endregion
-export { lazy };
-
-//# sourceMappingURL=lazy.js.map

package/out/replicache/src/lazy.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"lazy.js","names":[],"sources":["../../../../replicache/src/lazy.ts"],"sourcesContent":["export function lazy<T>(factory: () => T): () => T {\n  let value: T | undefined;\n  return () => {\n    if (value === undefined) {\n      value = factory();\n    }\n    return value;\n  };\n}\n"],"mappings":";AAAA,SAAgB,KAAQ,SAA2B;CACjD,IAAI;CACJ,aAAa;EACX,IAAI,UAAU,KAAA,GACZ,QAAQ,QAAQ;EAElB,OAAO;CACT;AACF"}

package/out/zero-cache/src/server/backup-watermark-reader.js

@@ -1,35 +0,0 @@
-import { must } from "../../../shared/src/must.js";
-import { parentWorker, shouldStartWorker, singleProcessMode } from "../types/processes.js";
-import { getZeroConfig } from "../config/zero-config.js";
-import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
-import { createLogContext } from "./logging.js";
-import { VfsBackupWatermarkReader, buildLitestreamVfsReplicaURL } from "../services/litestream/vfs-watermark-reader.js";
-import { VfsBackupWatermarkWorkerService } from "../services/litestream/vfs-watermark-worker.js";
-import { LogContext, consoleLogSink } from "@rocicorp/logger";
-//#region ../zero-cache/src/server/backup-watermark-reader.ts
-var lc = new LogContext("info", {}, consoleLogSink);
-async function runWorker(parent, env, ...argv) {
-	const config = getZeroConfig({
-		env,
-		argv
-	});
-	lc = createLogContext(config, "backup-watermark-reader");
-	const { litestream } = config;
-	const replicaURL = buildLitestreamVfsReplicaURL({
-		backupURL: must(litestream.backupURL, "Missing --litestream-backup-url for backup watermark reader"),
-		endpoint: litestream.endpoint,
-		region: litestream.region
-	});
-	const createReader = () => new VfsBackupWatermarkReader(lc, {
-		replicaURL,
-		extensionPath: litestream.vfsExtensionPath,
-		logLevel: litestream.logLevel,
-		logFile: litestream.vfsLogFile
-	});
-	await runUntilKilled(lc, parent ?? process, new VfsBackupWatermarkWorkerService(lc, parent, createReader, litestream.vfsProbeIntervalMs));
-}
-if (shouldStartWorker(parentWorker, singleProcessMode())) exitAfter(() => lc, () => runWorker(parentWorker, process.env, ...process.argv.slice(2)));
-//#endregion
-export { runWorker as default };
-
-//# sourceMappingURL=backup-watermark-reader.js.map

package/out/zero-cache/src/server/backup-watermark-reader.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"backup-watermark-reader.js","names":[],"sources":["../../../../../zero-cache/src/server/backup-watermark-reader.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getZeroConfig} from '../config/zero-config.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  buildLitestreamVfsReplicaURL,\n  VfsBackupWatermarkReader,\n} from '../services/litestream/vfs-watermark-reader.ts';\nimport {VfsBackupWatermarkWorkerService} from '../services/litestream/vfs-watermark-worker.ts';\nimport {\n  parentWorker,\n  shouldStartWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {createLogContext} from './logging.ts';\n\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker | null,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const config = getZeroConfig({env, argv});\n  lc = createLogContext(config, 'backup-watermark-reader');\n  const {litestream} = config;\n\n  const replicaURL = buildLitestreamVfsReplicaURL({\n    backupURL: must(\n      litestream.backupURL,\n      'Missing --litestream-backup-url for backup watermark reader',\n    ),\n    endpoint: litestream.endpoint,\n    region: litestream.region,\n  });\n\n  const createReader = () =>\n    new VfsBackupWatermarkReader(lc, {\n      replicaURL,\n      extensionPath: litestream.vfsExtensionPath,\n      logLevel: litestream.logLevel,\n      logFile: litestream.vfsLogFile,\n    });\n\n  await runUntilKilled(\n    lc,\n    parent ?? process,\n    new VfsBackupWatermarkWorkerService(\n      lc,\n      parent,\n      createReader,\n      litestream.vfsProbeIntervalMs,\n    ),\n  );\n}\n\n// Unlike the other workers, the backup watermark reader is *always* run as its\n// own OS process — forked on demand by the `VfsBackupMonitor` (via\n// `forkChildWorker`), or launched directly as a standalone debug tool — so that\n// the native Litestream VFS extension and its process-global `LITESTREAM_*` env\n// stay isolated from the rest of zero-cache. See `shouldStartWorker`.\nif (shouldStartWorker(parentWorker, singleProcessMode())) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(parentWorker, process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;AAiBA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,cAAc;EAAC;EAAK;CAAI,CAAC;CACxC,KAAK,iBAAiB,QAAQ,yBAAyB;CACvD,MAAM,EAAC,eAAc;CAErB,MAAM,aAAa,6BAA6B;EAC9C,WAAW,KACT,WAAW,WACX,6DACF;EACA,UAAU,WAAW;EACrB,QAAQ,WAAW;CACrB,CAAC;CAED,MAAM,qBACJ,IAAI,yBAAyB,IAAI;EAC/B;EACA,eAAe,WAAW;EAC1B,UAAU,WAAW;EACrB,SAAS,WAAW;CACtB,CAAC;CAEH,MAAM,eACJ,IACA,UAAU,SACV,IAAI,gCACF,IACA,QACA,cACA,WAAW,kBACb,CACF;AACF;AAOA,IAAI,kBAAkB,cAAc,kBAAkB,CAAC,GACrD,gBACQ,UACA,UAAU,cAAc,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CACrE"}

package/out/zero-cache/src/services/change-streamer/litestream3-backup-monitor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"litestream3-backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-backup-monitor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAOzD,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,eAAO,MAAM,iBAAiB,QAAS,CAAC;AAGxC;;;GAGG;AACH,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,wBAAwB,QAAc,CAAC;AAIpD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAG5D;;;;GAIG;AACH,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAEzD;;;;;;;GAOG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAOtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,wBAAyB,YAAW,aAAa;;IAC5D,QAAQ,CAAC,EAAE,oBAAoB;gBA8C7B,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,qBAAqB,EACrC,qBAAqB,EAAE,MAAM,EAC7B,iBAAiB,EAAE,mBAAmB;IAkBxC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAepB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC;IAgFvE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,kBAAkB,UAAO;IAwBxD,QAAQ,CAAC,iCAAiC,sBAgBxC;IA6RF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAkDtB"}

package/out/zero-cache/src/services/change-streamer/litestream3-backup-monitor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"litestream3-backup-monitor.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#changeStreamer","#state","#reservations","#watermarks","#verifyBackupState","#purgesBlocked","#fatal","#cleanupDelayMs","#runStartTime","#checkMetricsTimer","#initBackupLagMetric","#pushStatusWhenRestorable","#lastVerifiedUploadTime","#confirmRestorableBackup","#verifyBackupStateTimed","#checkWatermarks","#scheduleCleanup","#checkInitialBackupDeadline","#fetchWatermarks","#lastWatermark","#latestBackupTime","#maxWatermarkUpTo","#confirmedDurable","#backupStaleSince","#shutDownOnWedgedBackup","#initialBackupGraceMs","#shutDownOnMissingInitialBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-backup-monitor.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n} from '../../observability/metrics.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamBackupVerificationDuration,\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60_000;\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\n/**\n * Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n */\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/**\n * How long the actual backup state may remain *continuously* behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n *\n * A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes *some* such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n *\n * The change-log is conservatively *not* purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n */\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 * 60_000; // 15 minutes\n\nconst GiB = 1024 ** 3;\n\n/**\n * How long the BackupMonitor waits for the *first* restorable backup to appear\n * before concluding that the backup pipeline is broken and shutting the process\n * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).\n *\n * On a fresh stack the first backup is not restorable until the initial\n * Postgres->replica sync completes and litestream finishes uploading the\n * initial snapshot — a multipart upload that is not listable in the destination\n * until it is committed. The producer cannot observe that sub-snapshot progress,\n * so this is necessarily a generous timeout. It is scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance (~1 hour per\n * 50GB) so that it never fires before the platform would restart a view-syncer\n * that is waiting on the backup.\n */\nexport const INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 60_000; // 1 hour ...\nconst INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB; // ... per 50 GiB\n\n/**\n * How often the BackupMonitor re-checks whether the first restorable backup has\n * appeared while a view-syncer holds a snapshot reservation open during a cold\n * start.\n */\nexport const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10_000;\n\n/**\n * Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n *\n * See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n */\nexport type BackupStateVerifier = () => Promise<Date>;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The Litestream3BackupMonitor polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n *\n * Additionally, because the watermarks reported by litestream metrics\n * reflect what litestream *believes* has been backed up (which has been\n * observed to diverge from reality when uploads silently fail), the cleanup\n * watermark is only advanced after verifying it against the actual backup\n * state in the replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n */\nexport class Litestream3BackupMonitor implements BackupMonitor {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  readonly #verifyBackupState: BackupStateVerifier;\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup state ' +\n      '(as listed from the replica destination) could not be verified, or ' +\n      'is older than the backup progress claimed by litestream metrics. ' +\n      'A steadily increasing value indicates a wedged or failing backup. ' +\n      'The \"backup-wedged\" reason is emitted once just before the process ' +\n      'shuts itself down due to a persistently stale backup.',\n  });\n\n  // Rejected when the backup is determined to be genuinely wedged, which\n  // surfaces as a rejection of the `run()` promise and shuts the process down.\n  readonly #fatal = resolver<void>();\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  // The time of the most recent object actually verified in the backup\n  // destination, or `null` if none has ever been verified. Only assigned from a\n  // successful verifyBackupState() and never cleared, so a non-null value also\n  // means a restorable backup has been confirmed to exist (which is what gates\n  // the snapshot `status` signal and satisfies the initial-backup deadline).\n  #lastVerifiedUploadTime: Date | null = null;\n  // Epoch ms at which the actual backup state was first observed to be\n  // continuously behind the watermark litestream claims to have backed up,\n  // or `null` while the backup is keeping up. Reset whenever a purge is\n  // confirmed against a real upload.\n  #backupStaleSince: number | null = null;\n  // Epoch ms at which run() started, used to bound how long to wait for the\n  // first restorable backup to appear. `null` until run() is called.\n  #runStartTime: number | null = null;\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n    verifyBackupState: BackupStateVerifier,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#verifyBackupState = verifyBackupState;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#runStartTime = Date.now();\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    this.#initBackupLagMetric();\n    // Resolves on a normal stop; rejects if the backup is found to be wedged.\n    return Promise.race([this.#state.stopped(), this.#fatal.promise]);\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    // Note: the Subscription must be returned immediately so that the\n    //       websocket can begin sending liveness pings. The `status` signal\n    //       (which tells the view-syncer to restore) is withheld until a\n    //       restorable backup actually exists; see #pushStatusWhenRestorable.\n    void this.#pushStatusWhenRestorable(taskID, sub);\n    return sub;\n  }\n\n  /**\n   * Pushes the `status` snapshot signal once a restorable backup is confirmed\n   * to exist in the backup destination. On a warm start this is immediate; on a\n   * cold start (a fresh stack whose first backup is still being produced) the\n   * reservation is held open — kept alive by websocket liveness pings — and the\n   * view-syncer stays unready until the first backup lands, rather than being\n   * told to restore a backup that does not yet exist.\n   */\n  async #pushStatusWhenRestorable(\n    taskID: string,\n    sub: Subscription<SnapshotMessage>,\n  ): Promise<void> {\n    try {\n      while (this.#lastVerifiedUploadTime === null) {\n        if (!sub.active) {\n          return; // subscriber disconnected, or a newer reservation took over\n        }\n        if (await this.#confirmRestorableBackup()) {\n          break;\n        }\n        this.#lc.info?.(\n          `no restorable backup at ${this.#backupURL} yet; ` +\n            `holding ${taskID}'s reservation open until it lands`,\n        );\n        await sleep(RESTORABLE_BACKUP_POLL_INTERVAL_MS, this.#state.signal);\n      }\n      const changeLogState = await this.#changeStreamer.getChangeLogState();\n      if (sub.active) {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      }\n    } catch (e) {\n      if (this.#state.signal.aborted) {\n        return; // shutting down; not an error\n      }\n      this.#lc.warn?.(`failing snapshot reservation`, e);\n      sub.fail(e instanceof Error ? e : new Error(String(e)));\n    }\n  }\n\n  /**\n   * Verifies whether a restorable backup actually exists in the destination\n   * (as opposed to what litestream metrics merely claim). On success the\n   * verified upload time is cached — seeding the cleanup-verification fast path\n   * ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.\n   * Returns false if no backup is listable yet (e.g. the initial snapshot is\n   * still uploading).\n   */\n  async #confirmRestorableBackup(): Promise<boolean> {\n    try {\n      this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      return true;\n    } catch (e) {\n      this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);\n      return false;\n    }\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n    const duration = Date.now() - start.getTime();\n    litestreamSnapshotReservationDuration().recordMs(duration, {\n      ...litestreamMonitorMetricAttrs(this.#backupURL, 'legacy', 'view_syncer'),\n      result: updateCleanupDelay ? 'success' : 'cancelled',\n    });\n\n    if (updateCleanupDelay) {\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      await this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n    try {\n      await this.#checkInitialBackupDeadline();\n    } catch (e) {\n      this.#lc.warn?.(`error checking initial backup deadline`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  async #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    const maxWatermark = this.#maxWatermarkUpTo(latestCleanupTime);\n    if (maxWatermark.length === 0) {\n      return;\n    }\n    // Purge guard: the watermarks (and their backup times) come from\n    // litestream metrics, which are exported when litestream *believes*\n    // an upload succeeded, and have been observed to advance even when\n    // nothing was actually written to the backup destination. Purging the\n    // change-log based on a falsely advancing watermark permanently breaks\n    // the ability to restore + catch up. Before advancing the cleanup\n    // watermark, verify it against the actual backup state: a claimed\n    // backup time is only trusted if an object was actually uploaded to\n    // the replica destination at (or after) that time, modulo clock skew.\n    const claimedTime = must(this.#watermarks.get(maxWatermark));\n    if (!this.#confirmedDurable(claimedTime)) {\n      try {\n        this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      } catch (e) {\n        this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n        // Skipping the purge is safe: the change-log just grows.\n        this.#lc.warn?.(\n          `unable to verify backup state. skipping change-log cleanup ` +\n            `up to watermark ${maxWatermark} ` +\n            `(claimed backup time ${claimedTime.toISOString()})`,\n          e,\n        );\n        return;\n      }\n    }\n    // Watermarks whose backup time isn't yet confirmed durable remain in the\n    // map and are re-evaluated at the next check.\n    const lastUpload = must(this.#lastVerifiedUploadTime);\n    const verifiedWatermark = this.#maxWatermarkUpTo(\n      Math.min(\n        latestCleanupTime,\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n      ),\n    );\n    if (verifiedWatermark.length === 0) {\n      this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n      const now = Date.now();\n      if (this.#backupStaleSince === null) {\n        this.#backupStaleSince = now;\n      }\n      const staleForMs = now - this.#backupStaleSince;\n      this.#lc.warn?.(\n        `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n          `litestream claims it was backed up at ` +\n          `${claimedTime.toISOString()}, but the last object actually ` +\n          `uploaded to ${this.#backupURL} was at ` +\n          `${lastUpload.toISOString()}. ` +\n          `The backup has been stale for ${staleForMs} ms.`,\n      );\n      if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n        this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n      }\n      return;\n    }\n    // The cleanup watermark advanced past a real upload, so the backup is\n    // keeping up: clear the staleness clock.\n    this.#backupStaleSince = null;\n    this.#changeStreamer.scheduleCleanup(verifiedWatermark);\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= verifiedWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#lastWatermark = verifiedWatermark;\n  }\n\n  async #verifyBackupStateTimed(): Promise<Date> {\n    const start = performance.now();\n    let result: 'success' | 'error' = 'error';\n    try {\n      const ret = await this.#verifyBackupState();\n      result = 'success';\n      return ret;\n    } finally {\n      litestreamBackupVerificationDuration().recordMs(\n        performance.now() - start,\n        {\n          ...litestreamMonitorMetricAttrs(\n            this.#backupURL,\n            'legacy',\n            'replication_manager',\n          ),\n          result,\n        },\n      );\n    }\n  }\n\n  /**\n   * Called when the backup has been continuously stale for longer than\n   * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n   * backup, so the process is shut down by rejecting the {@link run()}\n   * promise. The exit is non-zero and logged at ERROR for alerting; on\n   * restart the monitor re-verifies, so a still-wedged backup keeps the\n   * process down until the destination is fixed.\n   */\n  #shutDownOnWedgedBackup(\n    staleForMs: number,\n    claimedTime: Date,\n    lastUpload: Date,\n  ) {\n    this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n    const err = new UnrecoverableError(\n      `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n        `at ${claimedTime.toISOString()}, but the last object actually ` +\n        `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n        `not advanced for ${staleForMs} ms (grace period ` +\n        `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n        `the backup; investigate the backup destination.`,\n    );\n    this.#lc.error?.(err.message);\n    clearInterval(this.#checkMetricsTimer);\n    this.#fatal.reject(err);\n  }\n\n  /**\n   * On a fresh stack, fails loudly if no restorable backup has appeared within\n   * a generous, replica-size-scaled grace period (see\n   * {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the \"give up\" decision\n   * onto the replication-manager — the producer responsible for creating the\n   * backup — instead of leaving view-syncers to wait (and the platform to keep\n   * restarting them) while the real fault is here. Does nothing once a\n   * restorable backup has been confirmed.\n   */\n  async #checkInitialBackupDeadline() {\n    if (this.#lastVerifiedUploadTime !== null || this.#runStartTime === null) {\n      return;\n    }\n    const elapsed = Date.now() - this.#runStartTime;\n    if (elapsed < this.#initialBackupGraceMs()) {\n      return;\n    }\n    // The deadline has elapsed without a confirmed backup. Do one definitive\n    // check against the destination before giving up, in case a backup landed\n    // but was never exercised by a reservation or a cleanup verification.\n    if (await this.#confirmRestorableBackup()) {\n      return;\n    }\n    this.#shutDownOnMissingInitialBackup(elapsed);\n  }\n\n  /**\n   * The grace period for the first restorable backup, scaled by replica size to\n   * mirror the platform's storage-scaled startup-probe allowance. Falls back to\n   * the minimum (one unit) if the replica file size cannot be determined.\n   */\n  #initialBackupGraceMs(): number {\n    let bytes = 0;\n    try {\n      bytes = statSync(this.#replicaFile).size;\n    } catch {\n      // Replica file not present/readable yet; fall back to the minimum grace.\n    }\n    const units = Math.max(\n      1,\n      Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES),\n    );\n    return units * INITIAL_BACKUP_GRACE_MS_PER_UNIT;\n  }\n\n  /**\n   * Called when no restorable backup has appeared within\n   * {@link #initialBackupGraceMs}. Continuing to run is pointless — no\n   * view-syncer can restore — and the fault is the backup pipeline, so the\n   * process exits non-zero with a logged error for alerting, mirroring\n   * {@link #shutDownOnWedgedBackup}.\n   */\n  #shutDownOnMissingInitialBackup(elapsedMs: number) {\n    const err = new UnrecoverableError(\n      `no restorable backup has appeared at ${this.#backupURL} within ` +\n        `${elapsedMs} ms of startup (grace period ` +\n        `${this.#initialBackupGraceMs()} ms). The initial backup pipeline ` +\n        `appears to be broken; shutting down so that the replication-manager ` +\n        `is flagged as the cause rather than the view-syncers waiting on it. ` +\n        `Investigate litestream replication to ${this.#backupURL}.`,\n    );\n    this.#lc.error?.(err.message);\n    clearInterval(this.#checkMetricsTimer);\n    this.#fatal.reject(err);\n  }\n\n  /**\n   * Returns the newest watermark whose backup time is at or before `cutoff`\n   * (epoch ms), or `''` if there is none.\n   */\n  #maxWatermarkUpTo(cutoff: number): string {\n    let max = '';\n    for (const [watermark, backupTime] of this.#watermarks) {\n      if (backupTime.getTime() <= cutoff && watermark > max) {\n        max = watermark;\n      }\n    }\n    return max;\n  }\n\n  /**\n   * Returns `true` if the actual backup state, as last verified against the\n   * backup destination, confirms that data claimed to be backed up at\n   * `claimedTime` is durable (i.e. an object was actually uploaded at or\n   * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n   * clock skew).\n   */\n  #confirmedDurable(claimedTime: Date): boolean {\n    const lastUpload = this.#lastVerifiedUploadTime;\n    return (\n      lastUpload !== null &&\n      claimedTime.getTime() <=\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n    );\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(async o => {\n      // For legacy litestream, we use the watermark metric (and its associated\n      // backup time) exported by litestream metrics to determine the time of\n      // of the backed up watermark. This is technically imprecise--it would be\n      // more correct to use the committed writeTimeMs--but it is good enough\n      // in that it serves the purpose of detecting a non-functioning backup.\n      // With litestream v5, this can be made more precise by querying the\n      // _zero.replicationState row from the backup directly using an LTX-based\n      // database reader.\n      const latestBackup = await this.#checkWatermarks();\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.getTime());\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAuBA,IAAa,oBAAoB;AACjC,IAAM,uBAAuB;;;;;AAM7B,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B,KAAK;AAE7C,IAAM,MAAM,QAAQ;;;;;;;;;;;;;;;AAgBpB,IAAa,mCAAmC,KAAK;AACrD,IAAM,kCAAkC,KAAK;;;;;;AAO7C,IAAa,qCAAqC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDlD,IAAa,2BAAb,MAA+D;CAC7D,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,gCAAyB,IAAI,IAAyB;CACtD,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAID,SAAkB,SAAe;CAEjC,iBAAyB;CACzB,oBAAiC;CAMjC,0BAAuC;CAKvC,oBAAmC;CAGnC,gBAA+B;CAC/B;CACA;CAEA,YACE,IACA,aACA,WACA,iBACA,gBACA,uBACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,eAAe;EACpB,KAAKC,aAAa;EAClB,KAAKC,mBAAmB;EACxB,KAAKC,kBAAkB;EACvB,KAAKI,qBAAqB;EAC1B,KAAKG,kBAAkB,KAAK,IAC1B,uBACA,oBACF;EAEA,KAAKX,IAAI,OACP,0BAA0B,sBAAsB,2BAClD;CACF;CAEA,MAAqB;EACnB,KAAKY,gBAAgB,KAAK,IAAI;EAC9B,KAAKZ,IAAI,OACP,yBAAyB,KAAKG,iBAAiB,QAC1C,KAAKQ,gBAAgB,kBAC5B;EACA,KAAKE,qBAAqB,YACxB,KAAK,mCACL,iBACF;EACA,KAAKC,qBAAqB;EAE1B,OAAO,QAAQ,KAAK,CAAC,KAAKT,OAAO,QAAQ,GAAG,KAAKK,OAAO,OAAO,CAAC;CAClE;CAEA,yBAAyB,QAA+C;EACtE,KAAKV,IAAI,OAAO,oCAAoC,OAAO,WAAW;EAEtE,KAAKM,cAAc,IAAI,MAAM,GAAG,IAAI,OAAO;EAE3C,MAAM,MAAM,aAAa,OAAwB,EAI/C,eAAe,KAAK,eAAe,QAAQ,KAAK,EAClD,CAAC;EACD,KAAKA,cAAc,IAAI,QAAQ;GAAC,uBAAO,IAAI,KAAK;GAAG;EAAG,CAAC;EAKvD,KAAUS,0BAA0B,QAAQ,GAAG;EAC/C,OAAO;CACT;;;;;;;;;CAUA,MAAMA,0BACJ,QACA,KACe;EACf,IAAI;GACF,OAAO,KAAKC,4BAA4B,MAAM;IAC5C,IAAI,CAAC,IAAI,QACP;IAEF,IAAI,MAAM,KAAKC,yBAAyB,GACtC;IAEF,KAAKjB,IAAI,OACP,2BAA2B,KAAKE,WAAW,gBAC9B,OAAO,mCACtB;IACA,MAAM,MAAM,oCAAoC,KAAKG,OAAO,MAAM;GACpE;GACA,MAAM,iBAAiB,MAAM,KAAKD,gBAAgB,kBAAkB;GACpE,IAAI,IAAI,QACN,IAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,KAAKF;IAAY,GAAG;GAAc,CAC/D,CAAC;EAEL,SAAS,GAAG;GACV,IAAI,KAAKG,OAAO,OAAO,SACrB;GAEF,KAAKL,IAAI,OAAO,gCAAgC,CAAC;GACjD,IAAI,KAAK,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC,CAAC;EACxD;CACF;;;;;;;;;CAUA,MAAMiB,2BAA6C;EACjD,IAAI;GACF,KAAKD,0BAA0B,MAAM,KAAKE,wBAAwB;GAClE,OAAO;EACT,SAAS,GAAG;GACV,KAAKlB,IAAI,OAAO,gCAAgC,KAAKE,cAAc,CAAC;GACpE,OAAO;EACT;CACF;CAEA,eAAe,QAAgB,qBAAqB,MAAM;EACxD,MAAM,MAAM,KAAKI,cAAc,IAAI,MAAM;EACzC,IAAI,QAAQ,KAAA,GACV;EAEF,KAAKA,cAAc,OAAO,MAAM;EAChC,MAAM,EAAC,OAAO,QAAO;EACrB,IAAI,OAAO;EACX,MAAM,WAAW,KAAK,IAAI,IAAI,MAAM,QAAQ;EAC5C,sCAAsC,EAAE,SAAS,UAAU;GACzD,GAAG,6BAA6B,KAAKJ,YAAY,UAAU,aAAa;GACxE,QAAQ,qBAAqB,YAAY;EAC3C,CAAC;EAED,IAAI,oBAAoB;GACtB,KAAKF,IAAI,OAAO,2BAA2B,OAAO,MAAM,SAAS,IAAI;GACrE,IAAI,WAAW,KAAKW,iBAAiB;IACnC,KAAKA,kBAAkB;IACvB,KAAKX,IAAI,OAAO,8BAA8B,SAAS,IAAI;GAC7D;EACF;CACF;CAGA,oCAA6C,YAAY;EACvD,IAAI;GACF,MAAM,KAAKmB,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKnB,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAKiB,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKpB,IAAI,OAAO,4BAA4B,CAAC;EAC/C;EACA,IAAI;GACF,MAAM,KAAKqB,4BAA4B;EACzC,SAAS,GAAG;GACV,KAAKrB,IAAI,OAAO,0CAA0C,CAAC;EAC7D;CACF;CAEA,OAAOsB,mBAIJ;EACD,MAAM,kBAAkB,KAAKnB;EAC7B,MAAM,SAAS,KAAKE,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAKL,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAKH,IAAI,OACP,8BAA8B,KAAKG,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GAErD,IAAI,WACF,MAAM;IAAC;IAAW;IAAM;GAAI;EAEhC;CAGN;CAEA,MAAMgB,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAKG,iBAAiB,GAChE,IAAI,YAAY,KAAKC,kBAAkB,CAAC,KAAKhB,YAAY,IAAI,SAAS,GAAG;GACvE,KAAKP,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAKO,YAAY,IAAI,WAAW,IAAI;GACpC,KAAKiB,oBAAoB;EAC3B;EAEF,OAAO,KAAKA;CACd;CAEA,MAAMJ,mBAAmB;EACvB,IAAI,KAAKd,cAAc,OAAO,GAAG;GAC/B,KAAKN,IAAI,OACP,6CAA6C,CAAC,GAAG,KAAKM,cAAc,KAAK,CAAC,GAC5E;GACA;EACF;EACA,MAAM,oBAAoB,KAAK,IAAI,IAAI,KAAKK;EAC5C,MAAM,eAAe,KAAKc,kBAAkB,iBAAiB;EAC7D,IAAI,aAAa,WAAW,GAC1B;EAWF,MAAM,cAAc,KAAK,KAAKlB,YAAY,IAAI,YAAY,CAAC;EAC3D,IAAI,CAAC,KAAKmB,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAKV,0BAA0B,MAAM,KAAKE,wBAAwB;EACpE,SAAS,GAAG;GACV,KAAKT,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAKT,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAKgB,uBAAuB;EACpD,MAAM,oBAAoB,KAAKS,kBAC7B,KAAK,IACH,mBACA,WAAW,QAAQ,IAAI,4BACzB,CACF;EACA,IAAI,kBAAkB,WAAW,GAAG;GAClC,KAAKhB,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAKkB,sBAAsB,MAC7B,KAAKA,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAKA;GAC9B,KAAK3B,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAKE,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAK0B,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAKD,oBAAoB;EACzB,KAAKvB,gBAAgB,gBAAgB,iBAAiB;EACtD,KAAK,MAAM,aAAa,KAAKG,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKgB,iBAAiB;CACxB;CAEA,MAAML,0BAAyC;EAC7C,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAKV,mBAAmB;GAC1C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,EAAE,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAKN,YACL,UACA,qBACF;IACA;GACF,CACF;EACF;CACF;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAKO,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAKP,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,cAAc,KAAKa,kBAAkB;EACrC,KAAKH,OAAO,OAAO,GAAG;CACxB;;;;;;;;;;CAWA,MAAMW,8BAA8B;EAClC,IAAI,KAAKL,4BAA4B,QAAQ,KAAKJ,kBAAkB,MAClE;EAEF,MAAM,UAAU,KAAK,IAAI,IAAI,KAAKA;EAClC,IAAI,UAAU,KAAKiB,sBAAsB,GACvC;EAKF,IAAI,MAAM,KAAKZ,yBAAyB,GACtC;EAEF,KAAKa,gCAAgC,OAAO;CAC9C;;;;;;CAOA,wBAAgC;EAC9B,IAAI,QAAQ;EACZ,IAAI;GACF,QAAQ,SAAS,KAAK7B,YAAY,EAAE;EACtC,QAAQ,CAER;EAKA,OAJc,KAAK,IACjB,GACA,KAAK,KAAK,QAAQ,+BAA+B,CAE5C,IAAQ;CACjB;;;;;;;;CASA,gCAAgC,WAAmB;EACjD,MAAM,MAAM,IAAI,mBACd,wCAAwC,KAAKC,WAAW,UACnD,UAAU,+BACV,KAAK2B,sBAAsB,EAAE,kNAGS,KAAK3B,WAAW,EAC7D;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,cAAc,KAAKa,kBAAkB;EACrC,KAAKH,OAAO,OAAO,GAAG;CACxB;;;;;CAMA,kBAAkB,QAAwB;EACxC,IAAI,MAAM;EACV,KAAK,MAAM,CAAC,WAAW,eAAe,KAAKH,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAChD,MAAM;EAGV,OAAO;CACT;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAKS;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;CAEA,OAAsB;EACpB,cAAc,KAAKH,kBAAkB;EACrC,KAAK,MAAM,EAAC,SAAQ,KAAKP,cAAc,OAAO,GAK5C,IAAI,OAAO;EAEb,KAAKD,OAAO,KAAK,KAAKL,GAAG;EACzB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MAAK;GASxB,MAAM,eAAe,MAAM,KAAKmB,iBAAiB;GACjD,IAAI,CAAC,cAAc;IACjB,KAAKnB,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKC,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,QAAQ,CAAC;IAClE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKD,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/replica-monitor.d.ts

@@ -1,16 +0,0 @@
-import type { LogContext } from '@rocicorp/logger';
-import type { Service } from '../service.ts';
-import type { ChangeStreamerService } from './change-streamer.ts';
-/**
- * The single-node equivalent of a backup monitor polls the replica file every
- * 30 seconds and schedules cleanup when the watermark
- * (i.e. stateVersion) moves forward.
- */
-export declare class ReplicaMonitor implements Service {
-    #private;
-    readonly id = "replica-monitor";
-    constructor(lc: LogContext, replicaFile: string, changeStreamer: ChangeStreamerService);
-    run(): Promise<void>;
-    stop(): Promise<void>;
-}
-//# sourceMappingURL=replica-monitor.d.ts.map

package/out/zero-cache/src/services/change-streamer/replica-monitor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"replica-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/replica-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAIhE;;;;GAIG;AACH,qBAAa,cAAe,YAAW,OAAO;;IAC5C,QAAQ,CAAC,EAAE,qBAAqB;gBAS9B,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,cAAc,EAAE,qBAAqB;IAOjC,GAAG;IAwBT,IAAI;CAIL"}

package/out/zero-cache/src/services/change-streamer/replica-monitor.js

@@ -1,54 +0,0 @@
-import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
-import { Database } from "../../../../zqlite/src/db.js";
-import { getReplicationState } from "../replicator/schema/replication-state.js";
-import { StatementRunner } from "../../db/statements.js";
-import { RunningState } from "../running-state.js";
-//#region ../zero-cache/src/services/change-streamer/replica-monitor.ts
-var CHECK_INTERVAL_MS = 30 * 1e3;
-/**
-* The single-node equivalent of a backup monitor polls the replica file every
-* 30 seconds and schedules cleanup when the watermark
-* (i.e. stateVersion) moves forward.
-*/
-var ReplicaMonitor = class {
-	id = "replica-monitor";
-	#lc;
-	#replicaFile;
-	#changeStreamer;
-	#state = new RunningState(this.id);
-	#lastWatermark = "";
-	constructor(lc, replicaFile, changeStreamer) {
-		this.#lc = lc.withContext("component", this.id);
-		this.#replicaFile = replicaFile;
-		this.#changeStreamer = changeStreamer;
-	}
-	async run() {
-		this.#lc.info?.(`starting replica monitor`);
-		await this.#state.sleep(CHECK_INTERVAL_MS);
-		while (this.#state.shouldRun()) {
-			const db = new Database(this.#lc, this.#replicaFile);
-			try {
-				const { stateVersion } = getReplicationState(new StatementRunner(db));
-				if (stateVersion !== this.#lastWatermark) {
-					this.#lastWatermark = stateVersion;
-					this.#lc.debug?.(`replicated up to watermark ${stateVersion}`);
-					this.#changeStreamer.scheduleCleanup(stateVersion);
-				}
-			} catch (e) {
-				this.#lc.error?.(`Unable to read watermark from replica`, e);
-			} finally {
-				db.close();
-			}
-			await this.#state.sleep(CHECK_INTERVAL_MS);
-		}
-		this.#lc.info?.(`replica monitor stopped`);
-	}
-	stop() {
-		this.#state.stop(this.#lc);
-		return promiseVoid;
-	}
-};
-//#endregion
-export { ReplicaMonitor };
-
-//# sourceMappingURL=replica-monitor.js.map

package/out/zero-cache/src/services/change-streamer/replica-monitor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"replica-monitor.js","names":["#lc","#replicaFile","#changeStreamer","#state","#lastWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/replica-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nconst CHECK_INTERVAL_MS = 30 * 1000;\n\n/**\n * The single-node equivalent of a backup monitor polls the replica file every\n * 30 seconds and schedules cleanup when the watermark\n * (i.e. stateVersion) moves forward.\n */\nexport class ReplicaMonitor implements Service {\n  readonly id = 'replica-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  #lastWatermark: string = '';\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  async run() {\n    this.#lc.info?.(`starting replica monitor`);\n    await this.#state.sleep(CHECK_INTERVAL_MS);\n\n    while (this.#state.shouldRun()) {\n      const db = new Database(this.#lc, this.#replicaFile);\n      try {\n        const {stateVersion} = getReplicationState(new StatementRunner(db));\n        if (stateVersion !== this.#lastWatermark) {\n          this.#lastWatermark = stateVersion;\n          this.#lc.debug?.(`replicated up to watermark ${stateVersion}`);\n          this.#changeStreamer.scheduleCleanup(stateVersion);\n        }\n      } catch (e) {\n        this.#lc.error?.(`Unable to read watermark from replica`, e);\n      } finally {\n        db.close();\n      }\n\n      await this.#state.sleep(CHECK_INTERVAL_MS);\n    }\n    this.#lc.info?.(`replica monitor stopped`);\n  }\n\n  stop() {\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;;AASA,IAAM,oBAAoB,KAAK;;;;;;AAO/B,IAAa,iBAAb,MAA+C;CAC7C,KAAc;CACd;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,iBAAyB;CAEzB,YACE,IACA,aACA,gBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,eAAe;EACpB,KAAKC,kBAAkB;CACzB;CAEA,MAAM,MAAM;EACV,KAAKF,IAAI,OAAO,0BAA0B;EAC1C,MAAM,KAAKG,OAAO,MAAM,iBAAiB;EAEzC,OAAO,KAAKA,OAAO,UAAU,GAAG;GAC9B,MAAM,KAAK,IAAI,SAAS,KAAKH,KAAK,KAAKC,YAAY;GACnD,IAAI;IACF,MAAM,EAAC,iBAAgB,oBAAoB,IAAI,gBAAgB,EAAE,CAAC;IAClE,IAAI,iBAAiB,KAAKG,gBAAgB;KACxC,KAAKA,iBAAiB;KACtB,KAAKJ,IAAI,QAAQ,8BAA8B,cAAc;KAC7D,KAAKE,gBAAgB,gBAAgB,YAAY;IACnD;GACF,SAAS,GAAG;IACV,KAAKF,IAAI,QAAQ,yCAAyC,CAAC;GAC7D,UAAU;IACR,GAAG,MAAM;GACX;GAEA,MAAM,KAAKG,OAAO,MAAM,iBAAiB;EAC3C;EACA,KAAKH,IAAI,OAAO,yBAAyB;CAC3C;CAEA,OAAO;EACL,KAAKG,OAAO,KAAK,KAAKH,GAAG;EACzB,OAAO;CACT;AACF"}

package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.d.ts

@@ -1,25 +0,0 @@
-import type { LogContext } from '@rocicorp/logger';
-import { Subscription } from '../../types/subscription.ts';
-import type { VfsBackupWatermark } from '../litestream/vfs-watermark-reader.ts';
-import type { BackupMonitor } from './backup-monitor.ts';
-import type { ChangeStreamerService } from './change-streamer.ts';
-import type { SnapshotMessage } from './snapshot.ts';
-export interface VfsBackupWatermarkSource {
-    readWatermark(): Promise<VfsBackupWatermark>;
-    close?(): void;
-}
-/**
- * Monitors a Litestream v0.5.x backup by reading Zero's replication state from
- * the backup itself through the Litestream SQLite VFS.
- */
-export declare class VfsBackupMonitor implements BackupMonitor {
-    #private;
-    readonly id = "vfs-backup-monitor";
-    constructor(lc: LogContext, backupURL: string, changeStreamer: ChangeStreamerService, initialCleanupDelayMs: number, probeIntervalMs: number, source: VfsBackupWatermarkSource);
-    run(): Promise<void>;
-    startSnapshotReservation(taskID: string): Subscription<SnapshotMessage>;
-    endReservation(taskID: string, updateCleanupDelay?: boolean): void;
-    readonly checkWatermarkAndScheduleCleanup: () => Promise<void>;
-    stop(): Promise<void>;
-}
-//# sourceMappingURL=vfs-backup-monitor.d.ts.map

package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"vfs-backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/vfs-backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAMzD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,uCAAuC,CAAC;AAE9E,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AASnD,MAAM,WAAW,wBAAwB;IACvC,aAAa,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,IAAI,CAAC;CAChB;AAED;;;GAGG;AACH,qBAAa,gBAAiB,YAAW,aAAa;;IACpD,QAAQ,CAAC,EAAE,wBAAwB;gBAuBjC,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,MAAM,EACjB,cAAc,EAAE,qBAAqB,EACrC,qBAAqB,EAAE,MAAM,EAC7B,eAAe,EAAE,MAAM,EACvB,MAAM,EAAE,wBAAwB;IAalC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAapB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC;IAwBvE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,kBAAkB,UAAO,GAAG,IAAI;IAwB/D,QAAQ,CAAC,gCAAgC,sBAcvC;IAwFF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAoCtB"}

package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.js

@@ -1,159 +0,0 @@
-import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
-import { getOrCreateCounter, getOrCreateGauge } from "../../observability/metrics.js";
-import { RunningState } from "../running-state.js";
-import { Subscription } from "../../types/subscription.js";
-import { litestreamBackupVerificationDuration, litestreamMonitorMetricAttrs, litestreamSnapshotReservationDuration } from "../litestream/metrics.js";
-//#region ../zero-cache/src/services/change-streamer/vfs-backup-monitor.ts
-var MIN_CLEANUP_DELAY_MS = 3e4;
-/**
-* Monitors a Litestream v0.5.x backup by reading Zero's replication state from
-* the backup itself through the Litestream SQLite VFS.
-*/
-var VfsBackupMonitor = class {
-	id = "vfs-backup-monitor";
-	#lc;
-	#backupURL;
-	#changeStreamer;
-	#source;
-	#probeIntervalMs;
-	#state = new RunningState(this.id);
-	#reservations = /* @__PURE__ */ new Map();
-	#watermarks = /* @__PURE__ */ new Map();
-	#purgesBlocked = getOrCreateCounter("replica", "purge_blocked", { description: "Number of change-log purges blocked because the actual backup watermark could not be read through the Litestream VFS." });
-	#lastWatermark = "";
-	#latestBackupWatermark;
-	#cleanupDelayMs;
-	#checkWatermarkTimer;
-	constructor(lc, backupURL, changeStreamer, initialCleanupDelayMs, probeIntervalMs, source) {
-		this.#lc = lc.withContext("component", this.id);
-		this.#backupURL = backupURL;
-		this.#changeStreamer = changeStreamer;
-		this.#source = source;
-		this.#probeIntervalMs = probeIntervalMs;
-		this.#cleanupDelayMs = Math.max(initialCleanupDelayMs, MIN_CLEANUP_DELAY_MS);
-	}
-	run() {
-		this.#lc.info?.(`monitoring v5 backups at ${this.#backupURL} with ${this.#cleanupDelayMs} ms cleanup delay`);
-		this.#checkWatermarkTimer = setInterval(this.checkWatermarkAndScheduleCleanup, this.#probeIntervalMs);
-		this.#initBackupLagMetric();
-		return this.#state.stopped();
-	}
-	startSnapshotReservation(taskID) {
-		this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);
-		this.#reservations.get(taskID)?.sub.cancel();
-		const sub = Subscription.create({ cleanup: () => this.endReservation(taskID, false) });
-		this.#reservations.set(taskID, {
-			start: /* @__PURE__ */ new Date(),
-			sub
-		});
-		this.#changeStreamer.getChangeLogState().then((changeLogState) => {
-			sub.push(["status", {
-				tag: "status",
-				backupURL: this.#backupURL,
-				...changeLogState
-			}]);
-		}).catch((e) => {
-			this.#lc.warn?.(`failing snapshot reservation`, e);
-			sub.fail(e);
-		});
-		return sub;
-	}
-	endReservation(taskID, updateCleanupDelay = true) {
-		const res = this.#reservations.get(taskID);
-		if (res === void 0) return;
-		this.#reservations.delete(taskID);
-		const { start, sub } = res;
-		sub.cancel();
-		const duration = Date.now() - start.getTime();
-		litestreamSnapshotReservationDuration().recordMs(duration, {
-			...litestreamMonitorMetricAttrs(this.#backupURL, "v5", "view_syncer"),
-			result: updateCleanupDelay ? "success" : "cancelled"
-		});
-		if (updateCleanupDelay) {
-			this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);
-			if (duration > this.#cleanupDelayMs) {
-				this.#cleanupDelayMs = duration;
-				this.#lc.info?.(`increased cleanup delay to ${duration} ms`);
-			}
-		}
-	}
-	checkWatermarkAndScheduleCleanup = async () => {
-		try {
-			await this.#checkWatermark();
-		} catch (e) {
-			this.#purgesBlocked.add(1, { reason: "vfs-probe-failed" });
-			this.#lc.warn?.("unable to read backup watermark through Litestream VFS. skipping change-log cleanup", e);
-			return;
-		}
-		this.#scheduleCleanup();
-	};
-	async #checkWatermark() {
-		const watermark = await this.#readWatermarkTimed();
-		this.#latestBackupWatermark = watermark;
-		if (watermark.watermark > this.#lastWatermark && !this.#watermarks.has(watermark.watermark)) {
-			this.#lc.info?.(`observed backup watermark=${watermark.watermark} through Litestream VFS at ${new Date(watermark.observedAtMs).toISOString()}`, {
-				writeTimeMs: watermark.writeTimeMs,
-				txid: watermark.txid,
-				lagSeconds: watermark.lagSeconds
-			});
-			this.#watermarks.set(watermark.watermark, watermark);
-		}
-	}
-	async #readWatermarkTimed() {
-		const start = performance.now();
-		let result = "error";
-		try {
-			const ret = await this.#source.readWatermark();
-			result = "success";
-			return ret;
-		} finally {
-			litestreamBackupVerificationDuration().recordMs(performance.now() - start, {
-				...litestreamMonitorMetricAttrs(this.#backupURL, "v5", "replication_manager"),
-				result
-			});
-		}
-	}
-	#scheduleCleanup() {
-		if (this.#reservations.size > 0) {
-			this.#lc.info?.(`watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`);
-			return;
-		}
-		const latestConfirmedWatermark = this.#latestBackupWatermark?.watermark;
-		if (latestConfirmedWatermark === void 0) return;
-		const maxWatermark = this.#maxWatermarkUpTo(Date.now() - this.#cleanupDelayMs, latestConfirmedWatermark);
-		if (maxWatermark.length === 0) return;
-		this.#changeStreamer.scheduleCleanup(maxWatermark);
-		for (const watermark of this.#watermarks.keys()) if (watermark <= maxWatermark) this.#watermarks.delete(watermark);
-		this.#lastWatermark = maxWatermark;
-	}
-	#maxWatermarkUpTo(cutoff, latestConfirmedWatermark) {
-		let max = "";
-		for (const [watermark, backupWatermark] of this.#watermarks) if (watermark <= latestConfirmedWatermark && backupWatermark.observedAtMs <= cutoff && watermark > max) max = watermark;
-		return max;
-	}
-	stop() {
-		clearInterval(this.#checkWatermarkTimer);
-		for (const { sub } of this.#reservations.values()) sub.cancel();
-		this.#source.close?.();
-		this.#state.stop(this.#lc);
-		return promiseVoid;
-	}
-	#initBackupLagMetric() {
-		getOrCreateGauge("replica", "backup_lag", {
-			description: "Latency from when a change is written to the replica to when it is backed up to litestream.",
-			unit: "millisecond"
-		}).addCallback((o) => {
-			const latestBackupWatermark = this.#latestBackupWatermark;
-			if (latestBackupWatermark?.writeTimeMs === void 0) {
-				this.#lc.warn?.(`no backed up watermarks. unable to report replica.backup_lag`);
-				return;
-			}
-			if (latestBackupWatermark.writeTimeMs === null) return;
-			o.observe(Math.max(0, latestBackupWatermark.observedAtMs - latestBackupWatermark.writeTimeMs));
-		});
-	}
-};
-//#endregion
-export { VfsBackupMonitor };
-
-//# sourceMappingURL=vfs-backup-monitor.js.map

package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"vfs-backup-monitor.js","names":["#lc","#backupURL","#changeStreamer","#source","#probeIntervalMs","#state","#reservations","#watermarks","#purgesBlocked","#cleanupDelayMs","#checkWatermarkTimer","#initBackupLagMetric","#checkWatermark","#scheduleCleanup","#readWatermarkTimed","#latestBackupWatermark","#lastWatermark","#maxWatermarkUpTo"],"sources":["../../../../../../zero-cache/src/services/change-streamer/vfs-backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n} from '../../observability/metrics.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamBackupVerificationDuration,\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {VfsBackupWatermark} from '../litestream/vfs-watermark-reader.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\nexport interface VfsBackupWatermarkSource {\n  readWatermark(): Promise<VfsBackupWatermark>;\n  close?(): void;\n}\n\n/**\n * Monitors a Litestream v0.5.x backup by reading Zero's replication state from\n * the backup itself through the Litestream SQLite VFS.\n */\nexport class VfsBackupMonitor implements BackupMonitor {\n  readonly id = 'vfs-backup-monitor';\n  readonly #lc: LogContext;\n  readonly #backupURL: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #source: VfsBackupWatermarkSource;\n  readonly #probeIntervalMs: number;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, VfsBackupWatermark>();\n\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup ' +\n      'watermark could not be read through the Litestream VFS.',\n  });\n\n  #lastWatermark: string = '';\n  #latestBackupWatermark: VfsBackupWatermark | undefined;\n  #cleanupDelayMs: number;\n  #checkWatermarkTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    backupURL: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n    probeIntervalMs: number,\n    source: VfsBackupWatermarkSource,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#backupURL = backupURL;\n    this.#changeStreamer = changeStreamer;\n    this.#source = source;\n    this.#probeIntervalMs = probeIntervalMs;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring v5 backups at ${this.#backupURL} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkWatermarkTimer = setInterval(\n      this.checkWatermarkAndScheduleCleanup,\n      this.#probeIntervalMs,\n    );\n    this.#initBackupLagMetric();\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true): void {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel();\n    const duration = Date.now() - start.getTime();\n    litestreamSnapshotReservationDuration().recordMs(duration, {\n      ...litestreamMonitorMetricAttrs(this.#backupURL, 'v5', 'view_syncer'),\n      result: updateCleanupDelay ? 'success' : 'cancelled',\n    });\n\n    if (updateCleanupDelay) {\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarkAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermark();\n    } catch (e) {\n      this.#purgesBlocked.add(1, {reason: 'vfs-probe-failed'});\n      this.#lc.warn?.(\n        `unable to read backup watermark through Litestream VFS. ` +\n          `skipping change-log cleanup`,\n        e,\n      );\n      return;\n    }\n\n    this.#scheduleCleanup();\n  };\n\n  async #checkWatermark(): Promise<void> {\n    const watermark = await this.#readWatermarkTimed();\n    this.#latestBackupWatermark = watermark;\n    if (\n      watermark.watermark > this.#lastWatermark &&\n      !this.#watermarks.has(watermark.watermark)\n    ) {\n      this.#lc.info?.(\n        `observed backup watermark=${watermark.watermark} through ` +\n          `Litestream VFS at ${new Date(watermark.observedAtMs).toISOString()}`,\n        {\n          writeTimeMs: watermark.writeTimeMs,\n          txid: watermark.txid,\n          lagSeconds: watermark.lagSeconds,\n        },\n      );\n      this.#watermarks.set(watermark.watermark, watermark);\n    }\n  }\n\n  async #readWatermarkTimed(): Promise<VfsBackupWatermark> {\n    const start = performance.now();\n    let result: 'success' | 'error' = 'error';\n    try {\n      const ret = await this.#source.readWatermark();\n      result = 'success';\n      return ret;\n    } finally {\n      litestreamBackupVerificationDuration().recordMs(\n        performance.now() - start,\n        {\n          ...litestreamMonitorMetricAttrs(\n            this.#backupURL,\n            'v5',\n            'replication_manager',\n          ),\n          result,\n        },\n      );\n    }\n  }\n\n  #scheduleCleanup(): void {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n\n    const latestConfirmedWatermark = this.#latestBackupWatermark?.watermark;\n    if (latestConfirmedWatermark === undefined) {\n      return;\n    }\n\n    const maxWatermark = this.#maxWatermarkUpTo(\n      Date.now() - this.#cleanupDelayMs,\n      latestConfirmedWatermark,\n    );\n    if (maxWatermark.length === 0) {\n      return;\n    }\n\n    this.#changeStreamer.scheduleCleanup(maxWatermark);\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= maxWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#lastWatermark = maxWatermark;\n  }\n\n  #maxWatermarkUpTo(cutoff: number, latestConfirmedWatermark: string): string {\n    let max = '';\n    for (const [watermark, backupWatermark] of this.#watermarks) {\n      if (\n        watermark <= latestConfirmedWatermark &&\n        backupWatermark.observedAtMs <= cutoff &&\n        watermark > max\n      ) {\n        max = watermark;\n      }\n    }\n    return max;\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkWatermarkTimer);\n    for (const {sub} of this.#reservations.values()) {\n      sub.cancel();\n    }\n    this.#source.close?.();\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric(): void {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackupWatermark = this.#latestBackupWatermark;\n      if (latestBackupWatermark?.writeTimeMs === undefined) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      if (latestBackupWatermark.writeTimeMs === null) {\n        return;\n      }\n      o.observe(\n        Math.max(\n          0,\n          latestBackupWatermark.observedAtMs -\n            latestBackupWatermark.writeTimeMs,\n        ),\n      );\n    });\n  }\n}\n"],"mappings":";;;;;;AAkBA,IAAM,uBAAuB;;;;;AAgB7B,IAAa,mBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,gCAAyB,IAAI,IAAyB;CACtD,8BAAuB,IAAI,IAAgC;CAE3D,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,wHAEJ,CAAC;CAED,iBAAyB;CACzB;CACA;CACA;CAEA,YACE,IACA,WACA,gBACA,uBACA,iBACA,QACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,aAAa;EAClB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,mBAAmB;EACxB,KAAKK,kBAAkB,KAAK,IAC1B,uBACA,oBACF;CACF;CAEA,MAAqB;EACnB,KAAKT,IAAI,OACP,4BAA4B,KAAKC,WAAW,QACvC,KAAKQ,gBAAgB,kBAC5B;EACA,KAAKC,uBAAuB,YAC1B,KAAK,kCACL,KAAKN,gBACP;EACA,KAAKO,qBAAqB;EAC1B,OAAO,KAAKN,OAAO,QAAQ;CAC7B;CAEA,yBAAyB,QAA+C;EACtE,KAAKL,IAAI,OAAO,oCAAoC,OAAO,WAAW;EACtE,KAAKM,cAAc,IAAI,MAAM,GAAG,IAAI,OAAO;EAE3C,MAAM,MAAM,aAAa,OAAwB,EAC/C,eAAe,KAAK,eAAe,QAAQ,KAAK,EAClD,CAAC;EACD,KAAKA,cAAc,IAAI,QAAQ;GAAC,uBAAO,IAAI,KAAK;GAAG;EAAG,CAAC;EAEvD,KAAUJ,gBACP,kBAAkB,EAClB,MAAK,mBAAkB;GACtB,IAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,KAAKD;IAAY,GAAG;GAAc,CAC/D,CAAC;EACH,CAAC,EACA,OAAM,MAAK;GACV,KAAKD,IAAI,OAAO,gCAAgC,CAAC;GACjD,IAAI,KAAK,CAAC;EACZ,CAAC;EACH,OAAO;CACT;CAEA,eAAe,QAAgB,qBAAqB,MAAY;EAC9D,MAAM,MAAM,KAAKM,cAAc,IAAI,MAAM;EACzC,IAAI,QAAQ,KAAA,GACV;EAEF,KAAKA,cAAc,OAAO,MAAM;EAChC,MAAM,EAAC,OAAO,QAAO;EACrB,IAAI,OAAO;EACX,MAAM,WAAW,KAAK,IAAI,IAAI,MAAM,QAAQ;EAC5C,sCAAsC,EAAE,SAAS,UAAU;GACzD,GAAG,6BAA6B,KAAKL,YAAY,MAAM,aAAa;GACpE,QAAQ,qBAAqB,YAAY;EAC3C,CAAC;EAED,IAAI,oBAAoB;GACtB,KAAKD,IAAI,OAAO,2BAA2B,OAAO,MAAM,SAAS,IAAI;GACrE,IAAI,WAAW,KAAKS,iBAAiB;IACnC,KAAKA,kBAAkB;IACvB,KAAKT,IAAI,OAAO,8BAA8B,SAAS,IAAI;GAC7D;EACF;CACF;CAGA,mCAA4C,YAAY;EACtD,IAAI;GACF,MAAM,KAAKY,gBAAgB;EAC7B,SAAS,GAAG;GACV,KAAKJ,eAAe,IAAI,GAAG,EAAC,QAAQ,mBAAkB,CAAC;GACvD,KAAKR,IAAI,OACP,uFAEA,CACF;GACA;EACF;EAEA,KAAKa,iBAAiB;CACxB;CAEA,MAAMD,kBAAiC;EACrC,MAAM,YAAY,MAAM,KAAKE,oBAAoB;EACjD,KAAKC,yBAAyB;EAC9B,IACE,UAAU,YAAY,KAAKC,kBAC3B,CAAC,KAAKT,YAAY,IAAI,UAAU,SAAS,GACzC;GACA,KAAKP,IAAI,OACP,6BAA6B,UAAU,UAAU,6BAC1B,IAAI,KAAK,UAAU,YAAY,EAAE,YAAY,KACpE;IACE,aAAa,UAAU;IACvB,MAAM,UAAU;IAChB,YAAY,UAAU;GACxB,CACF;GACA,KAAKO,YAAY,IAAI,UAAU,WAAW,SAAS;EACrD;CACF;CAEA,MAAMO,sBAAmD;EACvD,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAKX,QAAQ,cAAc;GAC7C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,EAAE,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAKF,YACL,MACA,qBACF;IACA;GACF,CACF;EACF;CACF;CAEA,mBAAyB;EACvB,IAAI,KAAKK,cAAc,OAAO,GAAG;GAC/B,KAAKN,IAAI,OACP,6CAA6C,CAAC,GAAG,KAAKM,cAAc,KAAK,CAAC,GAC5E;GACA;EACF;EAEA,MAAM,2BAA2B,KAAKS,wBAAwB;EAC9D,IAAI,6BAA6B,KAAA,GAC/B;EAGF,MAAM,eAAe,KAAKE,kBACxB,KAAK,IAAI,IAAI,KAAKR,iBAClB,wBACF;EACA,IAAI,aAAa,WAAW,GAC1B;EAGF,KAAKP,gBAAgB,gBAAgB,YAAY;EACjD,KAAK,MAAM,aAAa,KAAKK,YAAY,KAAK,GAC5C,IAAI,aAAa,cACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKS,iBAAiB;CACxB;CAEA,kBAAkB,QAAgB,0BAA0C;EAC1E,IAAI,MAAM;EACV,KAAK,MAAM,CAAC,WAAW,oBAAoB,KAAKT,aAC9C,IACE,aAAa,4BACb,gBAAgB,gBAAgB,UAChC,YAAY,KAEZ,MAAM;EAGV,OAAO;CACT;CAEA,OAAsB;EACpB,cAAc,KAAKG,oBAAoB;EACvC,KAAK,MAAM,EAAC,SAAQ,KAAKJ,cAAc,OAAO,GAC5C,IAAI,OAAO;EAEb,KAAKH,QAAQ,QAAQ;EACrB,KAAKE,OAAO,KAAK,KAAKL,GAAG;EACzB,OAAO;CACT;CAEA,uBAA6B;EAC3B,iBAAiB,WAAW,cAAc;GACxC,aACE;GAEF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,wBAAwB,KAAKe;GACnC,IAAI,uBAAuB,gBAAgB,KAAA,GAAW;IACpD,KAAKf,IAAI,OACP,8DACF;IACA;GACF;GACA,IAAI,sBAAsB,gBAAgB,MACxC;GAEF,EAAE,QACA,KAAK,IACH,GACA,sBAAsB,eACpB,sBAAsB,WAC1B,CACF;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.d.ts

@@ -1,44 +0,0 @@
-import type { LogContext, LogLevel } from '@rocicorp/logger';
-type LoadableDatabase = {
-    loadExtension(path: string): unknown;
-    close(): void;
-};
-type QueryDatabase = {
-    prepare(sql: string): {
-        get<T>(): T;
-    };
-    close(): void;
-};
-export type VfsBackupWatermark = {
-    watermark: string;
-    writeTimeMs: number | null;
-    txid: string;
-    lagSeconds: number;
-    observedAtMs: number;
-};
-export type VfsBackupWatermarkReaderOptions = {
-    readonly replicaURL: string;
-    readonly extensionPath: string;
-    readonly logLevel?: LogLevel | undefined;
-    readonly logFile?: string | undefined;
-    readonly databaseURI?: string | undefined;
-    readonly env?: NodeJS.ProcessEnv | undefined;
-};
-type VfsBackupWatermarkReaderDeps = {
-    readonly loaderFactory?: (() => LoadableDatabase) | undefined;
-    readonly databaseFactory?: ((lc: LogContext, uri: string) => QueryDatabase) | undefined;
-};
-export declare function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region, }: {
-    readonly backupURL: string;
-    readonly endpoint?: string | undefined;
-    readonly region?: string | undefined;
-}): string;
-export declare class VfsBackupWatermarkReader implements Disposable {
-    #private;
-    constructor(lc: LogContext, options: VfsBackupWatermarkReaderOptions, deps?: VfsBackupWatermarkReaderDeps);
-    readWatermark(): VfsBackupWatermark;
-    close(): void;
-    [Symbol.dispose](): void;
-}
-export {};
-//# sourceMappingURL=vfs-watermark-reader.d.ts.map

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"vfs-watermark-reader.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAE,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAQ3D,KAAK,gBAAgB,GAAG;IACtB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IACrC,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AAEF,KAAK,aAAa,GAAG;IACnB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG;QACpB,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;KACb,CAAC;IACF,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG;IAC5C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,KAAK,4BAA4B,GAAG;IAClC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,MAAM,gBAAgB,CAAC,GAAG,SAAS,CAAC;IAC9D,QAAQ,CAAC,eAAe,CAAC,EACrB,CAAC,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,KAAK,aAAa,CAAC,GAChD,SAAS,CAAC;CACf,CAAC;AASF,wBAAgB,4BAA4B,CAAC,EAC3C,SAAS,EACT,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,GAAG,MAAM,CAiBT;AAED,qBAAa,wBAAyB,YAAW,UAAU;;gBAQvD,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,+BAA+B,EACxC,IAAI,GAAE,4BAAiC;IAiCzC,aAAa,IAAI,kBAAkB;IAQnC,KAAK,IAAI,IAAI;IAKb,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAGzB"}

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.js

@@ -1,65 +0,0 @@
-import { parse, valita_exports } from "../../../../shared/src/valita.js";
-import { Database } from "../../../../zqlite/src/db.js";
-import Sqlite3Database from "@rocicorp/zero-sqlite3";
-//#region ../zero-cache/src/services/litestream/vfs-watermark-reader.ts
-var DEFAULT_VFS_DATABASE_URI = "file:zero-backup.db?vfs=litestream&mode=ro";
-var DEFAULT_VFS_LOG_LEVEL = "info";
-var backupWatermarkRowSchema = valita_exports.object({
-	watermark: valita_exports.string(),
-	writeTimeMs: valita_exports.number().nullable(),
-	txid: valita_exports.string(),
-	lagSeconds: valita_exports.number()
-});
-function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region }) {
-	if (endpoint === void 0 && region === void 0) return backupURL;
-	const url = new URL(backupURL);
-	if (url.protocol !== "s3:") return backupURL;
-	if (endpoint !== void 0 && !url.searchParams.has("endpoint")) url.searchParams.set("endpoint", endpoint);
-	if (region !== void 0 && !url.searchParams.has("region")) url.searchParams.set("region", region);
-	return url.toString();
-}
-var VfsBackupWatermarkReader = class {
-	#loader;
-	#db;
-	#stmt;
-	constructor(lc, options, deps = {}) {
-		const env = options.env ?? process.env;
-		env["LITESTREAM_REPLICA_URL"] = options.replicaURL;
-		env["LITESTREAM_LOG_LEVEL"] = (options.logLevel ?? DEFAULT_VFS_LOG_LEVEL).toUpperCase();
-		setOptionalEnv(env, "LITESTREAM_LOG_FILE", options.logFile);
-		const loaderFactory = deps.loaderFactory ?? (() => new Sqlite3Database(":memory:"));
-		const databaseFactory = deps.databaseFactory ?? ((lc, uri) => new Database(lc, uri, { readonly: true }));
-		this.#loader = loaderFactory();
-		this.#loader.loadExtension(options.extensionPath);
-		this.#db = databaseFactory(lc.withContext("component", "vfs-backup-watermark-reader"), options.databaseURI ?? DEFAULT_VFS_DATABASE_URI);
-		this.#stmt = this.#db.prepare(`
-      SELECT
-        stateVersion AS watermark,
-        writeTimeMs,
-        litestream_txid() AS txid,
-        litestream_lag() AS lagSeconds
-      FROM "_zero.replicationState"
-    `);
-	}
-	readWatermark() {
-		return {
-			...parse(this.#stmt.get(), backupWatermarkRowSchema),
-			observedAtMs: Date.now()
-		};
-	}
-	close() {
-		this.#db.close();
-		this.#loader.close();
-	}
-	[Symbol.dispose]() {
-		this.close();
-	}
-};
-function setOptionalEnv(env, name, value) {
-	if (value === void 0) delete env[name];
-	else env[name] = value;
-}
-//#endregion
-export { VfsBackupWatermarkReader, buildLitestreamVfsReplicaURL };
-
-//# sourceMappingURL=vfs-watermark-reader.js.map

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"vfs-watermark-reader.js","names":["#loader","#db","#stmt"],"sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-reader.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport Sqlite3Database from '@rocicorp/zero-sqlite3';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\n\nconst DEFAULT_VFS_DATABASE_URI = 'file:zero-backup.db?vfs=litestream&mode=ro';\nconst DEFAULT_VFS_LOG_LEVEL = 'info';\n\ntype LoadableDatabase = {\n  loadExtension(path: string): unknown;\n  close(): void;\n};\n\ntype QueryDatabase = {\n  prepare(sql: string): {\n    get<T>(): T;\n  };\n  close(): void;\n};\n\nexport type VfsBackupWatermark = {\n  watermark: string;\n  writeTimeMs: number | null;\n  txid: string;\n  lagSeconds: number;\n  observedAtMs: number;\n};\n\nexport type VfsBackupWatermarkReaderOptions = {\n  readonly replicaURL: string;\n  readonly extensionPath: string;\n  readonly logLevel?: LogLevel | undefined;\n  readonly logFile?: string | undefined;\n  readonly databaseURI?: string | undefined;\n  readonly env?: NodeJS.ProcessEnv | undefined;\n};\n\ntype VfsBackupWatermarkReaderDeps = {\n  readonly loaderFactory?: (() => LoadableDatabase) | undefined;\n  readonly databaseFactory?:\n    | ((lc: LogContext, uri: string) => QueryDatabase)\n    | undefined;\n};\n\nconst backupWatermarkRowSchema = v.object({\n  watermark: v.string(),\n  writeTimeMs: v.number().nullable(),\n  txid: v.string(),\n  lagSeconds: v.number(),\n});\n\nexport function buildLitestreamVfsReplicaURL({\n  backupURL,\n  endpoint,\n  region,\n}: {\n  readonly backupURL: string;\n  readonly endpoint?: string | undefined;\n  readonly region?: string | undefined;\n}): string {\n  if (endpoint === undefined && region === undefined) {\n    return backupURL;\n  }\n\n  const url = new URL(backupURL);\n  if (url.protocol !== 's3:') {\n    return backupURL;\n  }\n\n  if (endpoint !== undefined && !url.searchParams.has('endpoint')) {\n    url.searchParams.set('endpoint', endpoint);\n  }\n  if (region !== undefined && !url.searchParams.has('region')) {\n    url.searchParams.set('region', region);\n  }\n  return url.toString();\n}\n\nexport class VfsBackupWatermarkReader implements Disposable {\n  readonly #loader: LoadableDatabase;\n  readonly #db: QueryDatabase;\n  readonly #stmt: {\n    get<T>(): T;\n  };\n\n  constructor(\n    lc: LogContext,\n    options: VfsBackupWatermarkReaderOptions,\n    deps: VfsBackupWatermarkReaderDeps = {},\n  ) {\n    const env = options.env ?? process.env;\n    env['LITESTREAM_REPLICA_URL'] = options.replicaURL;\n    env['LITESTREAM_LOG_LEVEL'] = (\n      options.logLevel ?? DEFAULT_VFS_LOG_LEVEL\n    ).toUpperCase();\n    setOptionalEnv(env, 'LITESTREAM_LOG_FILE', options.logFile);\n\n    const loaderFactory =\n      deps.loaderFactory ?? (() => new Sqlite3Database(':memory:'));\n    const databaseFactory =\n      deps.databaseFactory ??\n      ((lc: LogContext, uri: string) =>\n        new Database(lc, uri, {readonly: true}));\n\n    this.#loader = loaderFactory();\n    this.#loader.loadExtension(options.extensionPath);\n\n    this.#db = databaseFactory(\n      lc.withContext('component', 'vfs-backup-watermark-reader'),\n      options.databaseURI ?? DEFAULT_VFS_DATABASE_URI,\n    );\n    this.#stmt = this.#db.prepare(/*sql*/ `\n      SELECT\n        stateVersion AS watermark,\n        writeTimeMs,\n        litestream_txid() AS txid,\n        litestream_lag() AS lagSeconds\n      FROM \"_zero.replicationState\"\n    `);\n  }\n\n  readWatermark(): VfsBackupWatermark {\n    const row = v.parse(this.#stmt.get(), backupWatermarkRowSchema);\n    return {\n      ...row,\n      observedAtMs: Date.now(),\n    };\n  }\n\n  close(): void {\n    this.#db.close();\n    this.#loader.close();\n  }\n\n  [Symbol.dispose](): void {\n    this.close();\n  }\n}\n\nfunction setOptionalEnv(\n  env: NodeJS.ProcessEnv,\n  name: string,\n  value: string | undefined,\n) {\n  if (value === undefined) {\n    delete env[name];\n  } else {\n    env[name] = value;\n  }\n}\n"],"mappings":";;;;AAKA,IAAM,2BAA2B;AACjC,IAAM,wBAAwB;AAsC9B,IAAM,2BAA2B,eAAE,OAAO;CACxC,WAAW,eAAE,OAAO;CACpB,aAAa,eAAE,OAAO,EAAE,SAAS;CACjC,MAAM,eAAE,OAAO;CACf,YAAY,eAAE,OAAO;AACvB,CAAC;AAED,SAAgB,6BAA6B,EAC3C,WACA,UACA,UAKS;CACT,IAAI,aAAa,KAAA,KAAa,WAAW,KAAA,GACvC,OAAO;CAGT,MAAM,MAAM,IAAI,IAAI,SAAS;CAC7B,IAAI,IAAI,aAAa,OACnB,OAAO;CAGT,IAAI,aAAa,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,UAAU,GAC5D,IAAI,aAAa,IAAI,YAAY,QAAQ;CAE3C,IAAI,WAAW,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,QAAQ,GACxD,IAAI,aAAa,IAAI,UAAU,MAAM;CAEvC,OAAO,IAAI,SAAS;AACtB;AAEA,IAAa,2BAAb,MAA4D;CAC1D;CACA;CACA;CAIA,YACE,IACA,SACA,OAAqC,CAAC,GACtC;EACA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EACnC,IAAI,4BAA4B,QAAQ;EACxC,IAAI,2BACF,QAAQ,YAAY,uBACpB,YAAY;EACd,eAAe,KAAK,uBAAuB,QAAQ,OAAO;EAE1D,MAAM,gBACJ,KAAK,wBAAwB,IAAI,gBAAgB,UAAU;EAC7D,MAAM,kBACJ,KAAK,qBACH,IAAgB,QAChB,IAAI,SAAS,IAAI,KAAK,EAAC,UAAU,KAAI,CAAC;EAE1C,KAAKA,UAAU,cAAc;EAC7B,KAAKA,QAAQ,cAAc,QAAQ,aAAa;EAEhD,KAAKC,MAAM,gBACT,GAAG,YAAY,aAAa,6BAA6B,GACzD,QAAQ,eAAe,wBACzB;EACA,KAAKC,QAAQ,KAAKD,IAAI,QAAgB;;;;;;;KAOrC;CACH;CAEA,gBAAoC;EAElC,OAAO;GACL,GAFU,MAAQ,KAAKC,MAAM,IAAI,GAAG,wBAEjC;GACH,cAAc,KAAK,IAAI;EACzB;CACF;CAEA,QAAc;EACZ,KAAKD,IAAI,MAAM;EACf,KAAKD,QAAQ,MAAM;CACrB;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,MAAM;CACb;AACF;AAEA,SAAS,eACP,KACA,MACA,OACA;CACA,IAAI,UAAU,KAAA,GACZ,OAAO,IAAI;MAEX,IAAI,QAAQ;AAEhB"}

package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.d.ts

@@ -1,11 +0,0 @@
-import type { LogContext } from '@rocicorp/logger';
-import type { Worker } from '../../types/processes.ts';
-import type { VfsBackupWatermark } from './vfs-watermark-reader.ts';
-export type VfsBackupWatermarkWorkerFactory = () => Worker;
-export declare class VfsBackupWatermarkWorkerSource {
-    #private;
-    constructor(lc: LogContext, workerFactory: VfsBackupWatermarkWorkerFactory, timeoutMs: number);
-    readWatermark(): Promise<VfsBackupWatermark>;
-    close(): void;
-}
-//# sourceMappingURL=vfs-watermark-worker-source.d.ts.map

package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"vfs-watermark-worker-source.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-worker-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,2BAA2B,CAAC;AAMlE,MAAM,MAAM,+BAA+B,GAAG,MAAM,MAAM,CAAC;AAE3D,qBAAa,8BAA8B;;gBASvC,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,+BAA+B,EAC9C,SAAS,EAAE,MAAM;IAUb,aAAa,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAclD,KAAK,IAAI,IAAI;CA4Dd"}