@rocicorp/zero 1.10.0-canary.2 → 1.10.0-canary.20

package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js

@@ -2,6 +2,7 @@ import { assert } from "../../../../shared/src/asserts.js";
 import { must } from "../../../../shared/src/must.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
 import { RunningState, UnrecoverableError } from "../running-state.js";
+import { versionFromLexi } from "../../types/lexi-version.js";
 import { Subscription } from "../../types/subscription.js";
 import { litestreamBackupVerificationDuration, litestreamMonitorMetricAttrs } from "../litestream/metrics.js";
 import { statSync } from "node:fs";
@@ -160,11 +161,16 @@ var Litestream3PrometheusPoller = class {
 			const watermark = metric.labels?.watermark;
 			const name = metric.labels?.name;
 			const time = /* @__PURE__ */ new Date(parseFloat(metric.value) * 1e3);
-			if (watermark) yield {
-				watermark,
-				time,
-				name
-			};
+			if (watermark) try {
+				versionFromLexi(watermark);
+				yield {
+					watermark,
+					time,
+					name
+				};
+			} catch (e) {
+				this.#lc.warn?.(`ignoring invalid watermark ${watermark}`, { metric }, e);
+			}
 		}
 	}
 	async #checkWatermarks() {

package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map

@@ -1 +1 @@
-{"version":3,"file":"litestream3-prometheus-poller.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#state","#stream","#watermarks","#verifyBackupState","#purgesBlocked","#checkMetricsTimer","#runStartTime","#confirmRestorableBackup","#lastVerifiedUploadTime","#verifyBackupStateTimed","#checkWatermarks","#publishVerifiedWatermarks","#checkInitialBackupDeadline","#fetchWatermarks","#lastWatermark","#latestBackupTime","#maxBackupBefore","#confirmedDurable","#backupStaleSince","#shutDownOnWedgedBackup","#initialBackupGraceMs","#shutDownOnMissingInitialBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamBackupVerificationDuration,\n  litestreamMonitorMetricAttrs,\n} from '../litestream/metrics.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nexport const CHECK_INTERVAL_MS = 10_000;\n\n/**\n * Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n */\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/**\n * How long the actual backup state may remain *continuously* behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n *\n * A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes *some* such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n *\n * The change-log is conservatively *not* purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n */\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 * 60_000; // 15 minutes\n\nconst GiB = 1024 ** 3;\n\n/**\n * How long the BackupMonitor waits for the *first* restorable backup to appear\n * before concluding that the backup pipeline is broken and shutting the process\n * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).\n *\n * On a fresh stack the first backup is not restorable until the initial\n * Postgres->replica sync completes and litestream finishes uploading the\n * initial snapshot — a multipart upload that is not listable in the destination\n * until it is committed. The producer cannot observe that sub-snapshot progress,\n * so this is necessarily a generous timeout. It is scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance (~1 hour per\n * 50GB) so that it never fires before the platform would restart a view-syncer\n * that is waiting on the backup.\n */\nexport const INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 60_000; // 1 hour ...\nconst INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB; // ... per 50 GiB\n\n/**\n * How often the BackupMonitor re-checks whether the first restorable backup has\n * appeared while a view-syncer holds a snapshot reservation open during a cold\n * start.\n */\nexport const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10_000;\n\n/**\n * Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n *\n * See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n */\nexport type BackupStateVerifier = () => Promise<Date>;\n\n/**\n * The Litestream3PrometheusPoller polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and emits verified watermarks to a stream of {@link BackedUpWatermark}s.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Because the watermarks reported by litestream metrics reflect what\n * litestream *believes* has been backed up (which has been observed to\n * diverge from reality when uploads silently fail), the cleanup watermark is\n * only published after verifying it against the actual backup state in the\n * replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n */\nexport class Litestream3PrometheusPoller {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #state = new RunningState('litestream3-prometheus-poller');\n\n  readonly #stream: Subscription<BackedUpWatermark>;\n  readonly #watermarks = new Map<string, Date>();\n\n  readonly #verifyBackupState: BackupStateVerifier;\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup state ' +\n      '(as listed from the replica destination) could not be verified, or ' +\n      'is older than the backup progress claimed by litestream metrics. ' +\n      'A steadily increasing value indicates a wedged or failing backup. ' +\n      'The \"backup-wedged\" reason is emitted once just before the process ' +\n      'shuts itself down due to a persistently stale backup.',\n  });\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  // The time of the most recent object actually verified in the backup\n  // destination, or `null` if none has ever been verified. Only assigned from a\n  // successful verifyBackupState() and never cleared, so a non-null value also\n  // means a restorable backup has been confirmed to exist (which is what gates\n  // the snapshot `status` signal and satisfies the initial-backup deadline).\n  #lastVerifiedUploadTime: Date | null = null;\n  // Epoch ms at which the actual backup state was first observed to be\n  // continuously behind the watermark litestream claims to have backed up,\n  // or `null` while the backup is keeping up. Reset whenever a purge is\n  // confirmed against a real upload.\n  #backupStaleSince: number | null = null;\n  // Epoch ms at which run() started, used to bound how long to wait for the\n  // first restorable backup to appear. `null` until run() is called.\n  #runStartTime: number | null = null;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    verifyBackupState: BackupStateVerifier,\n  ) {\n    this.#lc = lc.withContext('component', 'litestream3-prometheus-poller');\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#verifyBackupState = verifyBackupState;\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        clearInterval(this.#checkMetricsTimer);\n        this.#state.stop(this.#lc);\n      },\n    });\n  }\n\n  /**\n   * Starts polling for watermarks and pushes new ones to returned stream.\n   * Must only be called once.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#runStartTime === null, `Already called start()`);\n    this.#runStartTime = Date.now();\n    this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint}`);\n    this.#checkMetricsTimer = setInterval(\n      this.checkVerifyAndPublishWatermarks,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#stream;\n  }\n\n  /**\n   * Verifies whether a restorable backup actually exists in the destination\n   * (as opposed to what litestream metrics merely claim). On success the\n   * verified upload time is cached — seeding the cleanup-verification fast path\n   * ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.\n   * Returns false if no backup is listable yet (e.g. the initial snapshot is\n   * still uploading).\n   */\n  async #confirmRestorableBackup(): Promise<boolean> {\n    try {\n      this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      return true;\n    } catch (e) {\n      this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);\n      return false;\n    }\n  }\n\n  // Exported for testing\n  readonly checkVerifyAndPublishWatermarks = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      await this.#publishVerifiedWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n    try {\n      await this.#checkInitialBackupDeadline();\n    } catch (e) {\n      this.#lc.warn?.(`error checking initial backup deadline`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  async #publishVerifiedWatermarks() {\n    const maxBackup = this.#maxBackupBefore(Date.now());\n    if (!maxBackup) {\n      return;\n    }\n    const {watermark: maxWatermark, backupTime: claimedTime} = maxBackup;\n    // Purge guard: the watermarks (and their backup times) come from\n    // litestream metrics, which are exported when litestream *believes*\n    // an upload succeeded, and have been observed to advance even when\n    // nothing was actually written to the backup destination. Purging the\n    // change-log based on a falsely advancing watermark permanently breaks\n    // the ability to restore + catch up. Before advancing the cleanup\n    // watermark, verify it against the actual backup state: a claimed\n    // backup time is only trusted if an object was actually uploaded to\n    // the replica destination at (or after) that time, modulo clock skew.\n    if (!this.#confirmedDurable(claimedTime)) {\n      try {\n        this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      } catch (e) {\n        this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n        // Skipping the purge is safe: the change-log just grows.\n        this.#lc.warn?.(\n          `unable to verify backup state. skipping change-log cleanup ` +\n            `up to watermark ${maxWatermark} ` +\n            `(claimed backup time ${claimedTime.toISOString()})`,\n          e,\n        );\n        return;\n      }\n    }\n    // Watermarks whose backup time isn't yet confirmed durable remain in the\n    // map and are re-evaluated at the next check.\n    const lastUpload = must(this.#lastVerifiedUploadTime);\n    const verifiedBackup = this.#maxBackupBefore(\n      lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n    );\n    if (!verifiedBackup) {\n      this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n      const now = Date.now();\n      if (this.#backupStaleSince === null) {\n        this.#backupStaleSince = now;\n      }\n      const staleForMs = now - this.#backupStaleSince;\n      this.#lc.warn?.(\n        `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n          `litestream claims it was backed up at ` +\n          `${claimedTime.toISOString()}, but the last object actually ` +\n          `uploaded to ${this.#backupURL} was at ` +\n          `${lastUpload.toISOString()}. ` +\n          `The backup has been stale for ${staleForMs} ms.`,\n      );\n      if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n        this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n      }\n      return;\n    }\n    // The cleanup watermark advanced past a real upload, so the backup is\n    // keeping up: clear the staleness clock.\n    this.#backupStaleSince = null;\n    const {watermark: verifiedWatermark, backupTime} = verifiedBackup;\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= verifiedWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#stream.push({\n      watermark: verifiedWatermark,\n      backupTimeMs: backupTime.getTime(),\n    });\n    this.#lastWatermark = verifiedWatermark;\n  }\n\n  async #verifyBackupStateTimed(): Promise<Date> {\n    const start = performance.now();\n    let result: 'success' | 'error' = 'error';\n    try {\n      const ret = await this.#verifyBackupState();\n      result = 'success';\n      return ret;\n    } finally {\n      litestreamBackupVerificationDuration().recordMs(\n        performance.now() - start,\n        {\n          ...litestreamMonitorMetricAttrs(\n            this.#backupURL,\n            'legacy',\n            'replication_manager',\n          ),\n          result,\n        },\n      );\n    }\n  }\n\n  /**\n   * Called when the backup has been continuously stale for longer than\n   * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n   * backup, so the process is shut down by rejecting the {@link run()}\n   * promise. The exit is non-zero and logged at ERROR for alerting; on\n   * restart the monitor re-verifies, so a still-wedged backup keeps the\n   * process down until the destination is fixed.\n   */\n  #shutDownOnWedgedBackup(\n    staleForMs: number,\n    claimedTime: Date,\n    lastUpload: Date,\n  ) {\n    this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n    const err = new UnrecoverableError(\n      `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n        `at ${claimedTime.toISOString()}, but the last object actually ` +\n        `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n        `not advanced for ${staleForMs} ms (grace period ` +\n        `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n        `the backup; investigate the backup destination.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * On a fresh stack, fails loudly if no restorable backup has appeared within\n   * a generous, replica-size-scaled grace period (see\n   * {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the \"give up\" decision\n   * onto the replication-manager — the producer responsible for creating the\n   * backup — instead of leaving view-syncers to wait (and the platform to keep\n   * restarting them) while the real fault is here. Does nothing once a\n   * restorable backup has been confirmed.\n   */\n  async #checkInitialBackupDeadline() {\n    if (this.#lastVerifiedUploadTime !== null || this.#runStartTime === null) {\n      return;\n    }\n    const elapsed = Date.now() - this.#runStartTime;\n    if (elapsed < this.#initialBackupGraceMs()) {\n      return;\n    }\n    // The deadline has elapsed without a confirmed backup. Do one definitive\n    // check against the destination before giving up, in case a backup landed\n    // but was never exercised by a reservation or a cleanup verification.\n    if (await this.#confirmRestorableBackup()) {\n      return;\n    }\n    this.#shutDownOnMissingInitialBackup(elapsed);\n  }\n\n  /**\n   * The grace period for the first restorable backup, scaled by replica size to\n   * mirror the platform's storage-scaled startup-probe allowance. Falls back to\n   * the minimum (one unit) if the replica file size cannot be determined.\n   */\n  #initialBackupGraceMs(): number {\n    let bytes = 0;\n    try {\n      bytes = statSync(this.#replicaFile).size;\n    } catch {\n      // Replica file not present/readable yet; fall back to the minimum grace.\n    }\n    const units = Math.max(\n      1,\n      Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES),\n    );\n    return units * INITIAL_BACKUP_GRACE_MS_PER_UNIT;\n  }\n\n  /**\n   * Called when no restorable backup has appeared within\n   * {@link #initialBackupGraceMs}. Continuing to run is pointless — no\n   * view-syncer can restore — and the fault is the backup pipeline, so the\n   * process exits non-zero with a logged error for alerting, mirroring\n   * {@link #shutDownOnWedgedBackup}.\n   */\n  #shutDownOnMissingInitialBackup(elapsedMs: number) {\n    const err = new UnrecoverableError(\n      `no restorable backup has appeared at ${this.#backupURL} within ` +\n        `${elapsedMs} ms of startup (grace period ` +\n        `${this.#initialBackupGraceMs()} ms). The initial backup pipeline ` +\n        `appears to be broken; shutting down so that the replication-manager ` +\n        `is flagged as the cause rather than the view-syncers waiting on it. ` +\n        `Investigate litestream replication to ${this.#backupURL}.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * Returns the newest watermark whose backup time is at or before `cutoff`\n   * (epoch ms), or `''` if there is none.\n   */\n  #maxBackupBefore(\n    cutoff: number,\n  ): {watermark: string; backupTime: Date} | null {\n    let max = '';\n    let maxBackupTime: Date | undefined;\n    for (const [watermark, backupTime] of this.#watermarks) {\n      if (backupTime.getTime() <= cutoff && watermark > max) {\n        max = watermark;\n        maxBackupTime = backupTime;\n      }\n    }\n    if (!maxBackupTime) {\n      return null;\n    }\n    return {watermark: max, backupTime: maxBackupTime};\n  }\n\n  /**\n   * Returns `true` if the actual backup state, as last verified against the\n   * backup destination, confirms that data claimed to be backed up at\n   * `claimedTime` is durable (i.e. an object was actually uploaded at or\n   * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n   * clock skew).\n   */\n  #confirmedDurable(claimedTime: Date): boolean {\n    const lastUpload = this.#lastVerifiedUploadTime;\n    return (\n      lastUpload !== null &&\n      claimedTime.getTime() <=\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n    );\n  }\n}\n"],"mappings":";;;;;;;;;AAeA,IAAa,oBAAoB;;;;;AAMjC,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B,KAAK;AAE7C,IAAM,MAAM,QAAQ;;;;;;;;;;;;;;;AAgBpB,IAAa,mCAAmC,KAAK;AACrD,IAAM,kCAAkC,KAAK;;;;;;;;;;;;;;;;;AAmC7C,IAAa,8BAAb,MAAyC;CACvC;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,+BAA+B;CAElE;CACA,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAED,iBAAyB;CACzB,oBAAiC;CAMjC,0BAAuC;CAKvC,oBAAmC;CAGnC,gBAA+B;CAC/B;CAEA,YACE,IACA,aACA,WACA,iBACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,+BAA+B;EACtE,KAAKC,eAAe;EACpB,KAAKC,aAAa;EAClB,KAAKC,mBAAmB;EACxB,KAAKI,qBAAqB;EAC1B,KAAKF,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAKI,kBAAkB;GACrC,KAAKL,OAAO,KAAK,KAAKJ,GAAG;EAC3B,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKU,kBAAkB,MAAM,wBAAwB;EAC5D,KAAKA,gBAAgB,KAAK,IAAI;EAC9B,KAAKV,IAAI,OAAO,yBAAyB,KAAKG,kBAAkB;EAChE,KAAKM,qBAAqB,YACxB,KAAK,iCACL,iBACF;EACA,OAAO,KAAKJ;CACd;;;;;;;;;CAUA,MAAMM,2BAA6C;EACjD,IAAI;GACF,KAAKC,0BAA0B,MAAM,KAAKC,wBAAwB;GAClE,OAAO;EACT,SAAS,GAAG;GACV,KAAKb,IAAI,OAAO,gCAAgC,KAAKE,cAAc,CAAC;GACpE,OAAO;EACT;CACF;CAGA,kCAA2C,YAAY;EACrD,IAAI;GACF,MAAM,KAAKY,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKd,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAKY,2BAA2B;EACxC,SAAS,GAAG;GACV,KAAKf,IAAI,OAAO,4BAA4B,CAAC;EAC/C;EACA,IAAI;GACF,MAAM,KAAKgB,4BAA4B;EACzC,SAAS,GAAG;GACV,KAAKhB,IAAI,OAAO,0CAA0C,CAAC;EAC7D;CACF;CAEA,OAAOiB,mBAIJ;EACD,MAAM,kBAAkB,KAAKd;EAC7B,MAAM,SAAS,KAAKC,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAKJ,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAKH,IAAI,OACP,8BAA8B,KAAKG,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GAErD,IAAI,WACF,MAAM;IAAC;IAAW;IAAM;GAAI;EAEhC;CAGN;CAEA,MAAMW,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAKG,iBAAiB,GAChE,IAAI,YAAY,KAAKC,kBAAkB,CAAC,KAAKZ,YAAY,IAAI,SAAS,GAAG;GACvE,KAAKN,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAKM,YAAY,IAAI,WAAW,IAAI;GACpC,KAAKa,oBAAoB;EAC3B;EAEF,OAAO,KAAKA;CACd;CAEA,MAAMJ,6BAA6B;EACjC,MAAM,YAAY,KAAKK,iBAAiB,KAAK,IAAI,CAAC;EAClD,IAAI,CAAC,WACH;EAEF,MAAM,EAAC,WAAW,cAAc,YAAY,gBAAe;EAU3D,IAAI,CAAC,KAAKC,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAKT,0BAA0B,MAAM,KAAKC,wBAAwB;EACpE,SAAS,GAAG;GACV,KAAKL,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAKR,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAKY,uBAAuB;EACpD,MAAM,iBAAiB,KAAKQ,iBAC1B,WAAW,QAAQ,IAAI,4BACzB;EACA,IAAI,CAAC,gBAAgB;GACnB,KAAKZ,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAKc,sBAAsB,MAC7B,KAAKA,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAKA;GAC9B,KAAKtB,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAKE,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAKqB,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAKD,oBAAoB;EACzB,MAAM,EAAC,WAAW,mBAAmB,eAAc;EACnD,KAAK,MAAM,aAAa,KAAKhB,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKD,QAAQ,KAAK;GAChB,WAAW;GACX,cAAc,WAAW,QAAQ;EACnC,CAAC;EACD,KAAKa,iBAAiB;CACxB;CAEA,MAAML,0BAAyC;EAC7C,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAKN,mBAAmB;GAC1C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,EAAE,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAKL,YACL,UACA,qBACF;IACA;GACF,CACF;EACF;CACF;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAKM,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAKN,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;;;;;;CAWA,MAAMW,8BAA8B;EAClC,IAAI,KAAKJ,4BAA4B,QAAQ,KAAKF,kBAAkB,MAClE;EAEF,MAAM,UAAU,KAAK,IAAI,IAAI,KAAKA;EAClC,IAAI,UAAU,KAAKc,sBAAsB,GACvC;EAKF,IAAI,MAAM,KAAKb,yBAAyB,GACtC;EAEF,KAAKc,gCAAgC,OAAO;CAC9C;;;;;;CAOA,wBAAgC;EAC9B,IAAI,QAAQ;EACZ,IAAI;GACF,QAAQ,SAAS,KAAKxB,YAAY,EAAE;EACtC,QAAQ,CAER;EAKA,OAJc,KAAK,IACjB,GACA,KAAK,KAAK,QAAQ,+BAA+B,CAE5C,IAAQ;CACjB;;;;;;;;CASA,gCAAgC,WAAmB;EACjD,MAAM,MAAM,IAAI,mBACd,wCAAwC,KAAKC,WAAW,UACnD,UAAU,+BACV,KAAKsB,sBAAsB,EAAE,kNAGS,KAAKtB,WAAW,EAC7D;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;CAMA,iBACE,QAC8C;EAC9C,IAAI,MAAM;EACV,IAAI;EACJ,KAAK,MAAM,CAAC,WAAW,eAAe,KAAKC,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAAK;GACrD,MAAM;GACN,gBAAgB;EAClB;EAEF,IAAI,CAAC,eACH,OAAO;EAET,OAAO;GAAC,WAAW;GAAK,YAAY;EAAa;CACnD;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAKM;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;AACF"}
+{"version":3,"file":"litestream3-prometheus-poller.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#state","#stream","#watermarks","#verifyBackupState","#purgesBlocked","#checkMetricsTimer","#runStartTime","#confirmRestorableBackup","#lastVerifiedUploadTime","#verifyBackupStateTimed","#checkWatermarks","#publishVerifiedWatermarks","#checkInitialBackupDeadline","#fetchWatermarks","#lastWatermark","#latestBackupTime","#maxBackupBefore","#confirmedDurable","#backupStaleSince","#shutDownOnWedgedBackup","#initialBackupGraceMs","#shutDownOnMissingInitialBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamBackupVerificationDuration,\n  litestreamMonitorMetricAttrs,\n} from '../litestream/metrics.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nexport const CHECK_INTERVAL_MS = 10_000;\n\n/**\n * Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n */\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/**\n * How long the actual backup state may remain *continuously* behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n *\n * A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes *some* such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n *\n * The change-log is conservatively *not* purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n */\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 * 60_000; // 15 minutes\n\nconst GiB = 1024 ** 3;\n\n/**\n * How long the BackupMonitor waits for the *first* restorable backup to appear\n * before concluding that the backup pipeline is broken and shutting the process\n * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).\n *\n * On a fresh stack the first backup is not restorable until the initial\n * Postgres->replica sync completes and litestream finishes uploading the\n * initial snapshot — a multipart upload that is not listable in the destination\n * until it is committed. The producer cannot observe that sub-snapshot progress,\n * so this is necessarily a generous timeout. It is scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance (~1 hour per\n * 50GB) so that it never fires before the platform would restart a view-syncer\n * that is waiting on the backup.\n */\nexport const INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 60_000; // 1 hour ...\nconst INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB; // ... per 50 GiB\n\n/**\n * How often the BackupMonitor re-checks whether the first restorable backup has\n * appeared while a view-syncer holds a snapshot reservation open during a cold\n * start.\n */\nexport const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10_000;\n\n/**\n * Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n *\n * See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n */\nexport type BackupStateVerifier = () => Promise<Date>;\n\n/**\n * The Litestream3PrometheusPoller polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and emits verified watermarks to a stream of {@link BackedUpWatermark}s.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Because the watermarks reported by litestream metrics reflect what\n * litestream *believes* has been backed up (which has been observed to\n * diverge from reality when uploads silently fail), the cleanup watermark is\n * only published after verifying it against the actual backup state in the\n * replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n */\nexport class Litestream3PrometheusPoller {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #state = new RunningState('litestream3-prometheus-poller');\n\n  readonly #stream: Subscription<BackedUpWatermark>;\n  readonly #watermarks = new Map<string, Date>();\n\n  readonly #verifyBackupState: BackupStateVerifier;\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup state ' +\n      '(as listed from the replica destination) could not be verified, or ' +\n      'is older than the backup progress claimed by litestream metrics. ' +\n      'A steadily increasing value indicates a wedged or failing backup. ' +\n      'The \"backup-wedged\" reason is emitted once just before the process ' +\n      'shuts itself down due to a persistently stale backup.',\n  });\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  // The time of the most recent object actually verified in the backup\n  // destination, or `null` if none has ever been verified. Only assigned from a\n  // successful verifyBackupState() and never cleared, so a non-null value also\n  // means a restorable backup has been confirmed to exist (which is what gates\n  // the snapshot `status` signal and satisfies the initial-backup deadline).\n  #lastVerifiedUploadTime: Date | null = null;\n  // Epoch ms at which the actual backup state was first observed to be\n  // continuously behind the watermark litestream claims to have backed up,\n  // or `null` while the backup is keeping up. Reset whenever a purge is\n  // confirmed against a real upload.\n  #backupStaleSince: number | null = null;\n  // Epoch ms at which run() started, used to bound how long to wait for the\n  // first restorable backup to appear. `null` until run() is called.\n  #runStartTime: number | null = null;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    verifyBackupState: BackupStateVerifier,\n  ) {\n    this.#lc = lc.withContext('component', 'litestream3-prometheus-poller');\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#verifyBackupState = verifyBackupState;\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        clearInterval(this.#checkMetricsTimer);\n        this.#state.stop(this.#lc);\n      },\n    });\n  }\n\n  /**\n   * Starts polling for watermarks and pushes new ones to returned stream.\n   * Must only be called once.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#runStartTime === null, `Already called start()`);\n    this.#runStartTime = Date.now();\n    this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint}`);\n    this.#checkMetricsTimer = setInterval(\n      this.checkVerifyAndPublishWatermarks,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#stream;\n  }\n\n  /**\n   * Verifies whether a restorable backup actually exists in the destination\n   * (as opposed to what litestream metrics merely claim). On success the\n   * verified upload time is cached — seeding the cleanup-verification fast path\n   * ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.\n   * Returns false if no backup is listable yet (e.g. the initial snapshot is\n   * still uploading).\n   */\n  async #confirmRestorableBackup(): Promise<boolean> {\n    try {\n      this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      return true;\n    } catch (e) {\n      this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);\n      return false;\n    }\n  }\n\n  // Exported for testing\n  readonly checkVerifyAndPublishWatermarks = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      await this.#publishVerifiedWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n    try {\n      await this.#checkInitialBackupDeadline();\n    } catch (e) {\n      this.#lc.warn?.(`error checking initial backup deadline`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n          if (watermark) {\n            try {\n              versionFromLexi(watermark); // Sanity check: Ensure that it is valid.\n              yield {watermark, time, name};\n            } catch (e) {\n              this.#lc.warn?.(\n                `ignoring invalid watermark ${watermark}`,\n                {metric},\n                e,\n              );\n            }\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  async #publishVerifiedWatermarks() {\n    const maxBackup = this.#maxBackupBefore(Date.now());\n    if (!maxBackup) {\n      return;\n    }\n    const {watermark: maxWatermark, backupTime: claimedTime} = maxBackup;\n    // Purge guard: the watermarks (and their backup times) come from\n    // litestream metrics, which are exported when litestream *believes*\n    // an upload succeeded, and have been observed to advance even when\n    // nothing was actually written to the backup destination. Purging the\n    // change-log based on a falsely advancing watermark permanently breaks\n    // the ability to restore + catch up. Before advancing the cleanup\n    // watermark, verify it against the actual backup state: a claimed\n    // backup time is only trusted if an object was actually uploaded to\n    // the replica destination at (or after) that time, modulo clock skew.\n    if (!this.#confirmedDurable(claimedTime)) {\n      try {\n        this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      } catch (e) {\n        this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n        // Skipping the purge is safe: the change-log just grows.\n        this.#lc.warn?.(\n          `unable to verify backup state. skipping change-log cleanup ` +\n            `up to watermark ${maxWatermark} ` +\n            `(claimed backup time ${claimedTime.toISOString()})`,\n          e,\n        );\n        return;\n      }\n    }\n    // Watermarks whose backup time isn't yet confirmed durable remain in the\n    // map and are re-evaluated at the next check.\n    const lastUpload = must(this.#lastVerifiedUploadTime);\n    const verifiedBackup = this.#maxBackupBefore(\n      lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n    );\n    if (!verifiedBackup) {\n      this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n      const now = Date.now();\n      if (this.#backupStaleSince === null) {\n        this.#backupStaleSince = now;\n      }\n      const staleForMs = now - this.#backupStaleSince;\n      this.#lc.warn?.(\n        `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n          `litestream claims it was backed up at ` +\n          `${claimedTime.toISOString()}, but the last object actually ` +\n          `uploaded to ${this.#backupURL} was at ` +\n          `${lastUpload.toISOString()}. ` +\n          `The backup has been stale for ${staleForMs} ms.`,\n      );\n      if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n        this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n      }\n      return;\n    }\n    // The cleanup watermark advanced past a real upload, so the backup is\n    // keeping up: clear the staleness clock.\n    this.#backupStaleSince = null;\n    const {watermark: verifiedWatermark, backupTime} = verifiedBackup;\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= verifiedWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#stream.push({\n      watermark: verifiedWatermark,\n      backupTimeMs: backupTime.getTime(),\n    });\n    this.#lastWatermark = verifiedWatermark;\n  }\n\n  async #verifyBackupStateTimed(): Promise<Date> {\n    const start = performance.now();\n    let result: 'success' | 'error' = 'error';\n    try {\n      const ret = await this.#verifyBackupState();\n      result = 'success';\n      return ret;\n    } finally {\n      litestreamBackupVerificationDuration().recordMs(\n        performance.now() - start,\n        {\n          ...litestreamMonitorMetricAttrs(\n            this.#backupURL,\n            'legacy',\n            'replication_manager',\n          ),\n          result,\n        },\n      );\n    }\n  }\n\n  /**\n   * Called when the backup has been continuously stale for longer than\n   * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n   * backup, so the process is shut down by rejecting the {@link run()}\n   * promise. The exit is non-zero and logged at ERROR for alerting; on\n   * restart the monitor re-verifies, so a still-wedged backup keeps the\n   * process down until the destination is fixed.\n   */\n  #shutDownOnWedgedBackup(\n    staleForMs: number,\n    claimedTime: Date,\n    lastUpload: Date,\n  ) {\n    this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n    const err = new UnrecoverableError(\n      `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n        `at ${claimedTime.toISOString()}, but the last object actually ` +\n        `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n        `not advanced for ${staleForMs} ms (grace period ` +\n        `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n        `the backup; investigate the backup destination.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * On a fresh stack, fails loudly if no restorable backup has appeared within\n   * a generous, replica-size-scaled grace period (see\n   * {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the \"give up\" decision\n   * onto the replication-manager — the producer responsible for creating the\n   * backup — instead of leaving view-syncers to wait (and the platform to keep\n   * restarting them) while the real fault is here. Does nothing once a\n   * restorable backup has been confirmed.\n   */\n  async #checkInitialBackupDeadline() {\n    if (this.#lastVerifiedUploadTime !== null || this.#runStartTime === null) {\n      return;\n    }\n    const elapsed = Date.now() - this.#runStartTime;\n    if (elapsed < this.#initialBackupGraceMs()) {\n      return;\n    }\n    // The deadline has elapsed without a confirmed backup. Do one definitive\n    // check against the destination before giving up, in case a backup landed\n    // but was never exercised by a reservation or a cleanup verification.\n    if (await this.#confirmRestorableBackup()) {\n      return;\n    }\n    this.#shutDownOnMissingInitialBackup(elapsed);\n  }\n\n  /**\n   * The grace period for the first restorable backup, scaled by replica size to\n   * mirror the platform's storage-scaled startup-probe allowance. Falls back to\n   * the minimum (one unit) if the replica file size cannot be determined.\n   */\n  #initialBackupGraceMs(): number {\n    let bytes = 0;\n    try {\n      bytes = statSync(this.#replicaFile).size;\n    } catch {\n      // Replica file not present/readable yet; fall back to the minimum grace.\n    }\n    const units = Math.max(\n      1,\n      Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES),\n    );\n    return units * INITIAL_BACKUP_GRACE_MS_PER_UNIT;\n  }\n\n  /**\n   * Called when no restorable backup has appeared within\n   * {@link #initialBackupGraceMs}. Continuing to run is pointless — no\n   * view-syncer can restore — and the fault is the backup pipeline, so the\n   * process exits non-zero with a logged error for alerting, mirroring\n   * {@link #shutDownOnWedgedBackup}.\n   */\n  #shutDownOnMissingInitialBackup(elapsedMs: number) {\n    const err = new UnrecoverableError(\n      `no restorable backup has appeared at ${this.#backupURL} within ` +\n        `${elapsedMs} ms of startup (grace period ` +\n        `${this.#initialBackupGraceMs()} ms). The initial backup pipeline ` +\n        `appears to be broken; shutting down so that the replication-manager ` +\n        `is flagged as the cause rather than the view-syncers waiting on it. ` +\n        `Investigate litestream replication to ${this.#backupURL}.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * Returns the newest watermark whose backup time is at or before `cutoff`\n   * (epoch ms), or `''` if there is none.\n   */\n  #maxBackupBefore(\n    cutoff: number,\n  ): {watermark: string; backupTime: Date} | null {\n    let max = '';\n    let maxBackupTime: Date | undefined;\n    for (const [watermark, backupTime] of this.#watermarks) {\n      if (backupTime.getTime() <= cutoff && watermark > max) {\n        max = watermark;\n        maxBackupTime = backupTime;\n      }\n    }\n    if (!maxBackupTime) {\n      return null;\n    }\n    return {watermark: max, backupTime: maxBackupTime};\n  }\n\n  /**\n   * Returns `true` if the actual backup state, as last verified against the\n   * backup destination, confirms that data claimed to be backed up at\n   * `claimedTime` is durable (i.e. an object was actually uploaded at or\n   * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n   * clock skew).\n   */\n  #confirmedDurable(claimedTime: Date): boolean {\n    const lastUpload = this.#lastVerifiedUploadTime;\n    return (\n      lastUpload !== null &&\n      claimedTime.getTime() <=\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n    );\n  }\n}\n"],"mappings":";;;;;;;;;;AAgBA,IAAa,oBAAoB;;;;;AAMjC,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B,KAAK;AAE7C,IAAM,MAAM,QAAQ;;;;;;;;;;;;;;;AAgBpB,IAAa,mCAAmC,KAAK;AACrD,IAAM,kCAAkC,KAAK;;;;;;;;;;;;;;;;;AAmC7C,IAAa,8BAAb,MAAyC;CACvC;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,+BAA+B;CAElE;CACA,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAED,iBAAyB;CACzB,oBAAiC;CAMjC,0BAAuC;CAKvC,oBAAmC;CAGnC,gBAA+B;CAC/B;CAEA,YACE,IACA,aACA,WACA,iBACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,+BAA+B;EACtE,KAAKC,eAAe;EACpB,KAAKC,aAAa;EAClB,KAAKC,mBAAmB;EACxB,KAAKI,qBAAqB;EAC1B,KAAKF,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAKI,kBAAkB;GACrC,KAAKL,OAAO,KAAK,KAAKJ,GAAG;EAC3B,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKU,kBAAkB,MAAM,wBAAwB;EAC5D,KAAKA,gBAAgB,KAAK,IAAI;EAC9B,KAAKV,IAAI,OAAO,yBAAyB,KAAKG,kBAAkB;EAChE,KAAKM,qBAAqB,YACxB,KAAK,iCACL,iBACF;EACA,OAAO,KAAKJ;CACd;;;;;;;;;CAUA,MAAMM,2BAA6C;EACjD,IAAI;GACF,KAAKC,0BAA0B,MAAM,KAAKC,wBAAwB;GAClE,OAAO;EACT,SAAS,GAAG;GACV,KAAKb,IAAI,OAAO,gCAAgC,KAAKE,cAAc,CAAC;GACpE,OAAO;EACT;CACF;CAGA,kCAA2C,YAAY;EACrD,IAAI;GACF,MAAM,KAAKY,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKd,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAKY,2BAA2B;EACxC,SAAS,GAAG;GACV,KAAKf,IAAI,OAAO,4BAA4B,CAAC;EAC/C;EACA,IAAI;GACF,MAAM,KAAKgB,4BAA4B;EACzC,SAAS,GAAG;GACV,KAAKhB,IAAI,OAAO,0CAA0C,CAAC;EAC7D;CACF;CAEA,OAAOiB,mBAIJ;EACD,MAAM,kBAAkB,KAAKd;EAC7B,MAAM,SAAS,KAAKC,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAKJ,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAKH,IAAI,OACP,8BAA8B,KAAKG,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GACrD,IAAI,WACF,IAAI;IACF,gBAAgB,SAAS;IACzB,MAAM;KAAC;KAAW;KAAM;IAAI;GAC9B,SAAS,GAAG;IACV,KAAKH,IAAI,OACP,8BAA8B,aAC9B,EAAC,OAAM,GACP,CACF;GACF;EAEJ;CAGN;CAEA,MAAMc,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAKG,iBAAiB,GAChE,IAAI,YAAY,KAAKC,kBAAkB,CAAC,KAAKZ,YAAY,IAAI,SAAS,GAAG;GACvE,KAAKN,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAKM,YAAY,IAAI,WAAW,IAAI;GACpC,KAAKa,oBAAoB;EAC3B;EAEF,OAAO,KAAKA;CACd;CAEA,MAAMJ,6BAA6B;EACjC,MAAM,YAAY,KAAKK,iBAAiB,KAAK,IAAI,CAAC;EAClD,IAAI,CAAC,WACH;EAEF,MAAM,EAAC,WAAW,cAAc,YAAY,gBAAe;EAU3D,IAAI,CAAC,KAAKC,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAKT,0BAA0B,MAAM,KAAKC,wBAAwB;EACpE,SAAS,GAAG;GACV,KAAKL,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAKR,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAKY,uBAAuB;EACpD,MAAM,iBAAiB,KAAKQ,iBAC1B,WAAW,QAAQ,IAAI,4BACzB;EACA,IAAI,CAAC,gBAAgB;GACnB,KAAKZ,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAKc,sBAAsB,MAC7B,KAAKA,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAKA;GAC9B,KAAKtB,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAKE,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAKqB,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAKD,oBAAoB;EACzB,MAAM,EAAC,WAAW,mBAAmB,eAAc;EACnD,KAAK,MAAM,aAAa,KAAKhB,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKD,QAAQ,KAAK;GAChB,WAAW;GACX,cAAc,WAAW,QAAQ;EACnC,CAAC;EACD,KAAKa,iBAAiB;CACxB;CAEA,MAAML,0BAAyC;EAC7C,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAKN,mBAAmB;GAC1C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,EAAE,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAKL,YACL,UACA,qBACF;IACA;GACF,CACF;EACF;CACF;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAKM,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAKN,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;;;;;;CAWA,MAAMW,8BAA8B;EAClC,IAAI,KAAKJ,4BAA4B,QAAQ,KAAKF,kBAAkB,MAClE;EAEF,MAAM,UAAU,KAAK,IAAI,IAAI,KAAKA;EAClC,IAAI,UAAU,KAAKc,sBAAsB,GACvC;EAKF,IAAI,MAAM,KAAKb,yBAAyB,GACtC;EAEF,KAAKc,gCAAgC,OAAO;CAC9C;;;;;;CAOA,wBAAgC;EAC9B,IAAI,QAAQ;EACZ,IAAI;GACF,QAAQ,SAAS,KAAKxB,YAAY,EAAE;EACtC,QAAQ,CAER;EAKA,OAJc,KAAK,IACjB,GACA,KAAK,KAAK,QAAQ,+BAA+B,CAE5C,IAAQ;CACjB;;;;;;;;CASA,gCAAgC,WAAmB;EACjD,MAAM,MAAM,IAAI,mBACd,wCAAwC,KAAKC,WAAW,UACnD,UAAU,+BACV,KAAKsB,sBAAsB,EAAE,kNAGS,KAAKtB,WAAW,EAC7D;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;CAMA,iBACE,QAC8C;EAC9C,IAAI,MAAM;EACV,IAAI;EACJ,KAAK,MAAM,CAAC,WAAW,eAAe,KAAKC,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAAK;GACrD,MAAM;GACN,gBAAgB;EAClB;EAEF,IAAI,CAAC,eACH,OAAO;EAET,OAAO;GAAC,WAAW;GAAK,YAAY;EAAa;CACnD;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAKM;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;AACF"}

package/out/zero-cache/src/services/change-streamer/shard-sampling.d.ts

@@ -0,0 +1,11 @@
+import type { ShardID } from '../../types/shards.ts';
+/**
+ * Stable percentage sampling keyed by shard and an arbitrary identity: a
+ * subscriber's task for read routing, a watermark for catchup comparison.
+ *
+ * The same key always lands in the same bucket, so raising `percent` only ever
+ * adds to the sampled set, and every process in a shard agrees on the set
+ * without coordinating.
+ */
+export declare function isSampledForShard(shard: ShardID, key: string, percent: number): boolean;
+//# sourceMappingURL=shard-sampling.d.ts.map

package/out/zero-cache/src/services/change-streamer/shard-sampling.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shard-sampling.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/shard-sampling.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAEnD;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,GACd,OAAO,CAQT"}

package/out/zero-cache/src/services/change-streamer/shard-sampling.js

@@ -0,0 +1,19 @@
+import { h32 } from "../../../../shared/src/hash.js";
+//#region ../zero-cache/src/services/change-streamer/shard-sampling.ts
+/**
+* Stable percentage sampling keyed by shard and an arbitrary identity: a
+* subscriber's task for read routing, a watermark for catchup comparison.
+*
+* The same key always lands in the same bucket, so raising `percent` only ever
+* adds to the sampled set, and every process in a shard agrees on the set
+* without coordinating.
+*/
+function isSampledForShard(shard, key, percent) {
+	if (percent >= 100) return true;
+	if (percent <= 0) return false;
+	return h32(`${shard.appID}/${shard.shardNum}:${key}`) % 100 < percent;
+}
+//#endregion
+export { isSampledForShard };
+
+//# sourceMappingURL=shard-sampling.js.map

package/out/zero-cache/src/services/change-streamer/shard-sampling.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shard-sampling.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/shard-sampling.ts"],"sourcesContent":["import {h32} from '../../../../shared/src/hash.ts';\nimport type {ShardID} from '../../types/shards.ts';\n\n/**\n * Stable percentage sampling keyed by shard and an arbitrary identity: a\n * subscriber's task for read routing, a watermark for catchup comparison.\n *\n * The same key always lands in the same bucket, so raising `percent` only ever\n * adds to the sampled set, and every process in a shard agrees on the set\n * without coordinating.\n */\nexport function isSampledForShard(\n  shard: ShardID,\n  key: string,\n  percent: number,\n): boolean {\n  if (percent >= 100) {\n    return true;\n  }\n  if (percent <= 0) {\n    return false;\n  }\n  return h32(`${shard.appID}/${shard.shardNum}:${key}`) % 100 < percent;\n}\n"],"mappings":";;;;;;;;;;AAWA,SAAgB,kBACd,OACA,KACA,SACS;CACT,IAAI,WAAW,KACb,OAAO;CAET,IAAI,WAAW,GACb,OAAO;CAET,OAAO,IAAI,GAAG,MAAM,MAAM,GAAG,MAAM,SAAS,GAAG,KAAK,IAAI,MAAM;AAChE"}

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts

@@ -2,13 +2,23 @@ import type { LogContext } from '@rocicorp/logger';
 import type { Source } from '../../types/streams.ts';
 import type { BackupConfig } from './change-streamer-service.ts';
 import type { SnapshotMessage } from './snapshot.ts';
+import type { ChangeLogReadSource } from './sqlite-change-log-read-router.ts';
 export declare class SnapshotReservations {
     #private;
     constructor(lc: LogContext, backupConfig: BackupConfig, onClose?: ((taskID: string) => void) | undefined);
     open(taskID: string): Source<SnapshotMessage>;
     close(taskID: string): void;
+    isCurrent(taskID: string, source: Source<SnapshotMessage>): boolean;
     confirmationsRequired(): boolean;
-    confirm(replicaVersion: string, minWatermark: string): void;
+    unconfirmedTaskIDs(): string[];
+    /** Confirms one reservation with the bounds of its pinned read source. */
+    confirmFor(taskID: string, replicaVersion: string, minWatermark: string, source: ChangeLogReadSource): void;
+    /**
+     * Notes that a confirmation was deferred, returning true only the first time
+     * for this reservation. Confirmation is retried on every backup, so an
+     * undeduplicated count would measure backups rather than delayed followers.
+     */
+    noteConfirmationDelayed(taskID: string): boolean;
     getReservedWatermarks(): string[];
 }
 //# sourceMappingURL=snapshot-reservations.d.ts.map

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-reservations.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAMnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,8BAA8B,CAAC;AAC/D,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,qBAAa,oBAAoB;;gBAO7B,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS;IAOlD,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC;IAY7C,KAAK,CAAC,MAAM,EAAE,MAAM;IA+BpB,qBAAqB;IASrB,OAAO,CAAC,cAAc,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM;IAWpD,qBAAqB;CAMtB"}
+{"version":3,"file":"snapshot-reservations.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAOnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,8BAA8B,CAAC;AAC/D,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,oCAAoC,CAAC;AAE5E,qBAAa,oBAAoB;;gBAO7B,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS;IAOlD,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC;IAY7C,KAAK,CAAC,MAAM,EAAE,MAAM;IAIpB,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC,GAAG,OAAO;IAwCnE,qBAAqB;IASrB,kBAAkB,IAAI,MAAM,EAAE;IAM9B,0EAA0E;IAC1E,UAAU,CACR,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,mBAAmB,GAC1B,IAAI;IAgBP;;;;OAIG;IACH,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAIhD,qBAAqB;CAMtB"}

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js

@@ -1,5 +1,5 @@
 import { Subscription } from "../../types/subscription.js";
-import { litestreamMonitorMetricAttrs, litestreamSnapshotReservationDuration } from "../litestream/metrics.js";
+import { litestreamMonitorMetricAttrs, litestreamSnapshotReservationConfirmDuration, litestreamSnapshotReservationDuration } from "../litestream/metrics.js";
 //#region ../zero-cache/src/services/change-streamer/snapshot-reservations.ts
 var SnapshotReservations = class {
 	#lc;
@@ -22,6 +22,12 @@ var SnapshotReservations = class {
 	close(taskID) {
 		this.#close(taskID, void 0);
 	}
+	isCurrent(taskID, source) {
+		return this.#reservations.get(taskID)?.owns(source) ?? false;
+	}
+	#metricAttrs() {
+		return litestreamMonitorMetricAttrs(this.#backupConfig.backupURL, this.#backupConfig.litestreamVersion, "view_syncer");
+	}
 	#close(taskID, cancelledInstanceID) {
 		const res = this.#reservations.get(taskID);
 		if (res && (!cancelledInstanceID || res.instanceID === cancelledInstanceID)) {
@@ -31,8 +37,9 @@ var SnapshotReservations = class {
 			const duration = Date.now() - res.startTime.getTime();
 			this.#lc.info?.(`ended snapshot reservation for ${taskID} (${duration} ms)`);
 			litestreamSnapshotReservationDuration().recordMs(duration, {
-				...litestreamMonitorMetricAttrs(this.#backupConfig.backupURL, this.#backupConfig.litestreamVersion, "view_syncer"),
-				result: cancelledInstanceID || !res.confirmed() ? "cancelled" : "closed"
+				...this.#metricAttrs(),
+				result: cancelledInstanceID ? "cancelled" : "closed",
+				confirmed: res.confirmed()
 			});
 		}
 	}
@@ -40,12 +47,29 @@ var SnapshotReservations = class {
 		for (const res of this.#reservations.values()) if (!res.confirmed()) return true;
 		return false;
 	}
-	confirm(replicaVersion, minWatermark) {
-		for (const [taskID, res] of this.#reservations.entries()) if (!res.confirmed()) {
+	unconfirmedTaskIDs() {
+		return [...this.#reservations.entries()].filter(([, reservation]) => !reservation.confirmed()).map(([taskID]) => taskID);
+	}
+	/** Confirms one reservation with the bounds of its pinned read source. */
+	confirmFor(taskID, replicaVersion, minWatermark, source) {
+		const res = this.#reservations.get(taskID);
+		if (res && !res.confirmed()) {
 			this.#lc.info?.(`reserving change-log entries since ${minWatermark} for ${taskID}`);
 			res.confirm(this.#backupConfig.backupURL, replicaVersion, minWatermark);
+			litestreamSnapshotReservationConfirmDuration().recordMs(Date.now() - res.startTime.getTime(), {
+				...this.#metricAttrs(),
+				source
+			});
 		}
 	}
+	/**
+	* Notes that a confirmation was deferred, returning true only the first time
+	* for this reservation. Confirmation is retried on every backup, so an
+	* undeduplicated count would measure backups rather than delayed followers.
+	*/
+	noteConfirmationDelayed(taskID) {
+		return this.#reservations.get(taskID)?.noteDelayed() ?? false;
+	}
 	getReservedWatermarks() {
 		return Array.from(this.#reservations.values(), ({ reservedWatermark }) => reservedWatermark).filter((watermark) => watermark !== null);
 	}
@@ -55,6 +79,7 @@ var Reservation = class {
 	startTime = /* @__PURE__ */ new Date();
 	#downstream;
 	#watermark = null;
+	#delayNoted = false;
 	constructor(instanceID, downstream) {
 		this.instanceID = instanceID;
 		this.#downstream = downstream;
@@ -65,6 +90,14 @@ var Reservation = class {
 	confirmed() {
 		return this.#watermark !== null;
 	}
+	owns(source) {
+		return this.#downstream === source;
+	}
+	noteDelayed() {
+		if (this.#delayNoted) return false;
+		this.#delayNoted = true;
+		return true;
+	}
 	confirm(backupURL, replicaVersion, minWatermark) {
 		if (this.#watermark === null) {
 			if (this.#downstream.active) this.#downstream.push(["status", {

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-reservations.js","names":["#lc","#backupConfig","#onClose","#reservations","#close","#downstream","#watermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {BackupConfig} from './change-streamer-service.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport class SnapshotReservations {\n  readonly #lc: LogContext;\n  readonly #backupConfig: BackupConfig;\n  readonly #onClose: ((taskID: string) => void) | undefined;\n  readonly #reservations = new Map<string, Reservation>();\n\n  constructor(\n    lc: LogContext,\n    backupConfig: BackupConfig,\n    onClose?: ((taskID: string) => void) | undefined,\n  ) {\n    this.#lc = lc.withContext('component', 'snapshot-reserver');\n    this.#backupConfig = backupConfig;\n    this.#onClose = onClose;\n  }\n\n  open(taskID: string): Source<SnapshotMessage> {\n    this.close(taskID);\n\n    const instanceID = {};\n    const downstream = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.#close(taskID, instanceID),\n    });\n    this.#reservations.set(taskID, new Reservation(instanceID, downstream));\n    this.#lc.info?.(`created snasphot reservation for ${taskID}`);\n    return downstream;\n  }\n\n  close(taskID: string) {\n    this.#close(taskID, undefined);\n  }\n\n  #close(taskID: string, cancelledInstanceID: InstanceID | undefined) {\n    const res = this.#reservations.get(taskID);\n    if (\n      res &&\n      (!cancelledInstanceID || res.instanceID === cancelledInstanceID)\n    ) {\n      // Note: delete first, so that the reservation is gone when close() is called.\n      this.#reservations.delete(taskID);\n      this.#onClose?.(taskID);\n      res.close();\n\n      const duration = Date.now() - res.startTime.getTime();\n      this.#lc.info?.(\n        `ended snapshot reservation for ${taskID} (${duration} ms)`,\n      );\n      litestreamSnapshotReservationDuration().recordMs(duration, {\n        ...litestreamMonitorMetricAttrs(\n          this.#backupConfig.backupURL,\n          this.#backupConfig.litestreamVersion,\n          'view_syncer',\n        ),\n        result:\n          cancelledInstanceID || !res.confirmed() ? 'cancelled' : 'closed',\n      });\n    }\n  }\n\n  confirmationsRequired() {\n    for (const res of this.#reservations.values()) {\n      if (!res.confirmed()) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  confirm(replicaVersion: string, minWatermark: string) {\n    for (const [taskID, res] of this.#reservations.entries()) {\n      if (!res.confirmed()) {\n        this.#lc.info?.(\n          `reserving change-log entries since ${minWatermark} for ${taskID}`,\n        );\n        res.confirm(this.#backupConfig.backupURL, replicaVersion, minWatermark);\n      }\n    }\n  }\n\n  getReservedWatermarks() {\n    return Array.from(\n      this.#reservations.values(),\n      ({reservedWatermark}) => reservedWatermark,\n    ).filter(watermark => watermark !== null);\n  }\n}\n\ntype InstanceID = {};\n\nclass Reservation {\n  readonly instanceID: InstanceID;\n  readonly startTime: Date = new Date();\n  readonly #downstream: Subscription<SnapshotMessage>;\n  #watermark: string | null = null;\n\n  constructor(\n    instanceID: InstanceID,\n    downstream: Subscription<SnapshotMessage>,\n  ) {\n    this.instanceID = instanceID;\n    this.#downstream = downstream;\n  }\n\n  get reservedWatermark() {\n    return this.#watermark;\n  }\n\n  confirmed() {\n    return this.#watermark !== null;\n  }\n\n  confirm(backupURL: string, replicaVersion: string, minWatermark: string) {\n    if (this.#watermark === null) {\n      if (this.#downstream.active) {\n        this.#downstream.push([\n          'status',\n          {tag: 'status', backupURL, replicaVersion, minWatermark},\n        ]);\n      }\n      this.#watermark = minWatermark;\n    }\n  }\n\n  close() {\n    this.#downstream.cancel();\n  }\n}\n"],"mappings":";;;AAUA,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA,gCAAyB,IAAI,IAAyB;CAEtD,YACE,IACA,cACA,SACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKC,gBAAgB;EACrB,KAAKC,WAAW;CAClB;CAEA,KAAK,QAAyC;EAC5C,KAAK,MAAM,MAAM;EAEjB,MAAM,aAAa,CAAC;EACpB,MAAM,aAAa,aAAa,OAAwB,EACtD,eAAe,KAAKE,OAAO,QAAQ,UAAU,EAC/C,CAAC;EACD,KAAKD,cAAc,IAAI,QAAQ,IAAI,YAAY,YAAY,UAAU,CAAC;EACtE,KAAKH,IAAI,OAAO,oCAAoC,QAAQ;EAC5D,OAAO;CACT;CAEA,MAAM,QAAgB;EACpB,KAAKI,OAAO,QAAQ,KAAA,CAAS;CAC/B;CAEA,OAAO,QAAgB,qBAA6C;EAClE,MAAM,MAAM,KAAKD,cAAc,IAAI,MAAM;EACzC,IACE,QACC,CAAC,uBAAuB,IAAI,eAAe,sBAC5C;GAEA,KAAKA,cAAc,OAAO,MAAM;GAChC,KAAKD,WAAW,MAAM;GACtB,IAAI,MAAM;GAEV,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ;GACpD,KAAKF,IAAI,OACP,kCAAkC,OAAO,IAAI,SAAS,KACxD;GACA,sCAAsC,EAAE,SAAS,UAAU;IACzD,GAAG,6BACD,KAAKC,cAAc,WACnB,KAAKA,cAAc,mBACnB,aACF;IACA,QACE,uBAAuB,CAAC,IAAI,UAAU,IAAI,cAAc;GAC5D,CAAC;EACH;CACF;CAEA,wBAAwB;EACtB,KAAK,MAAM,OAAO,KAAKE,cAAc,OAAO,GAC1C,IAAI,CAAC,IAAI,UAAU,GACjB,OAAO;EAGX,OAAO;CACT;CAEA,QAAQ,gBAAwB,cAAsB;EACpD,KAAK,MAAM,CAAC,QAAQ,QAAQ,KAAKA,cAAc,QAAQ,GACrD,IAAI,CAAC,IAAI,UAAU,GAAG;GACpB,KAAKH,IAAI,OACP,sCAAsC,aAAa,OAAO,QAC5D;GACA,IAAI,QAAQ,KAAKC,cAAc,WAAW,gBAAgB,YAAY;EACxE;CAEJ;CAEA,wBAAwB;EACtB,OAAO,MAAM,KACX,KAAKE,cAAc,OAAO,IACzB,EAAC,wBAAuB,iBAC3B,EAAE,QAAO,cAAa,cAAc,IAAI;CAC1C;AACF;AAIA,IAAM,cAAN,MAAkB;CAChB;CACA,4BAA2B,IAAI,KAAK;CACpC;CACA,aAA4B;CAE5B,YACE,YACA,YACA;EACA,KAAK,aAAa;EAClB,KAAKE,cAAc;CACrB;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAKC;CACd;CAEA,YAAY;EACV,OAAO,KAAKA,eAAe;CAC7B;CAEA,QAAQ,WAAmB,gBAAwB,cAAsB;EACvE,IAAI,KAAKA,eAAe,MAAM;GAC5B,IAAI,KAAKD,YAAY,QACnB,KAAKA,YAAY,KAAK,CACpB,UACA;IAAC,KAAK;IAAU;IAAW;IAAgB;GAAY,CACzD,CAAC;GAEH,KAAKC,aAAa;EACpB;CACF;CAEA,QAAQ;EACN,KAAKD,YAAY,OAAO;CAC1B;AACF"}
+{"version":3,"file":"snapshot-reservations.js","names":["#lc","#backupConfig","#onClose","#reservations","#close","#metricAttrs","#downstream","#watermark","#delayNoted"],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationConfirmDuration,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {BackupConfig} from './change-streamer-service.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\nimport type {ChangeLogReadSource} from './sqlite-change-log-read-router.ts';\n\nexport class SnapshotReservations {\n  readonly #lc: LogContext;\n  readonly #backupConfig: BackupConfig;\n  readonly #onClose: ((taskID: string) => void) | undefined;\n  readonly #reservations = new Map<string, Reservation>();\n\n  constructor(\n    lc: LogContext,\n    backupConfig: BackupConfig,\n    onClose?: ((taskID: string) => void) | undefined,\n  ) {\n    this.#lc = lc.withContext('component', 'snapshot-reserver');\n    this.#backupConfig = backupConfig;\n    this.#onClose = onClose;\n  }\n\n  open(taskID: string): Source<SnapshotMessage> {\n    this.close(taskID);\n\n    const instanceID = {};\n    const downstream = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.#close(taskID, instanceID),\n    });\n    this.#reservations.set(taskID, new Reservation(instanceID, downstream));\n    this.#lc.info?.(`created snasphot reservation for ${taskID}`);\n    return downstream;\n  }\n\n  close(taskID: string) {\n    this.#close(taskID, undefined);\n  }\n\n  isCurrent(taskID: string, source: Source<SnapshotMessage>): boolean {\n    return this.#reservations.get(taskID)?.owns(source) ?? false;\n  }\n\n  #metricAttrs() {\n    return litestreamMonitorMetricAttrs(\n      this.#backupConfig.backupURL,\n      this.#backupConfig.litestreamVersion,\n      'view_syncer',\n    );\n  }\n\n  #close(taskID: string, cancelledInstanceID: InstanceID | undefined) {\n    const res = this.#reservations.get(taskID);\n    if (\n      res &&\n      (!cancelledInstanceID || res.instanceID === cancelledInstanceID)\n    ) {\n      // Note: delete first, so that the reservation is gone when close() is called.\n      this.#reservations.delete(taskID);\n      this.#onClose?.(taskID);\n      res.close();\n\n      const duration = Date.now() - res.startTime.getTime();\n      this.#lc.info?.(\n        `ended snapshot reservation for ${taskID} (${duration} ms)`,\n      );\n      // `result` reports how the reservation ended and `confirmed` whether it\n      // ever received its bounds. They are separate dimensions: a follower\n      // that gives up while waiting for a confirmation is\n      // `result=cancelled, confirmed=false`, which a single collapsed\n      // attribute cannot distinguish from a client that simply went away.\n      litestreamSnapshotReservationDuration().recordMs(duration, {\n        ...this.#metricAttrs(),\n        result: cancelledInstanceID ? 'cancelled' : 'closed',\n        confirmed: res.confirmed(),\n      });\n    }\n  }\n\n  confirmationsRequired() {\n    for (const res of this.#reservations.values()) {\n      if (!res.confirmed()) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  unconfirmedTaskIDs(): string[] {\n    return [...this.#reservations.entries()]\n      .filter(([, reservation]) => !reservation.confirmed())\n      .map(([taskID]) => taskID);\n  }\n\n  /** Confirms one reservation with the bounds of its pinned read source. */\n  confirmFor(\n    taskID: string,\n    replicaVersion: string,\n    minWatermark: string,\n    source: ChangeLogReadSource,\n  ): void {\n    const res = this.#reservations.get(taskID);\n    if (res && !res.confirmed()) {\n      this.#lc.info?.(\n        `reserving change-log entries since ${minWatermark} for ${taskID}`,\n      );\n      res.confirm(this.#backupConfig.backupURL, replicaVersion, minWatermark);\n      // Measured from `open()`, not from the first confirmation attempt: what\n      // matters is how long the follower waited before it could restore.\n      litestreamSnapshotReservationConfirmDuration().recordMs(\n        Date.now() - res.startTime.getTime(),\n        {...this.#metricAttrs(), source},\n      );\n    }\n  }\n\n  /**\n   * Notes that a confirmation was deferred, returning true only the first time\n   * for this reservation. Confirmation is retried on every backup, so an\n   * undeduplicated count would measure backups rather than delayed followers.\n   */\n  noteConfirmationDelayed(taskID: string): boolean {\n    return this.#reservations.get(taskID)?.noteDelayed() ?? false;\n  }\n\n  getReservedWatermarks() {\n    return Array.from(\n      this.#reservations.values(),\n      ({reservedWatermark}) => reservedWatermark,\n    ).filter(watermark => watermark !== null);\n  }\n}\n\ntype InstanceID = {};\n\nclass Reservation {\n  readonly instanceID: InstanceID;\n  readonly startTime: Date = new Date();\n  readonly #downstream: Subscription<SnapshotMessage>;\n  #watermark: string | null = null;\n  #delayNoted = false;\n\n  constructor(\n    instanceID: InstanceID,\n    downstream: Subscription<SnapshotMessage>,\n  ) {\n    this.instanceID = instanceID;\n    this.#downstream = downstream;\n  }\n\n  get reservedWatermark() {\n    return this.#watermark;\n  }\n\n  confirmed() {\n    return this.#watermark !== null;\n  }\n\n  owns(source: Source<SnapshotMessage>): boolean {\n    return this.#downstream === source;\n  }\n\n  noteDelayed(): boolean {\n    if (this.#delayNoted) {\n      return false;\n    }\n    this.#delayNoted = true;\n    return true;\n  }\n\n  confirm(backupURL: string, replicaVersion: string, minWatermark: string) {\n    if (this.#watermark === null) {\n      if (this.#downstream.active) {\n        this.#downstream.push([\n          'status',\n          {tag: 'status', backupURL, replicaVersion, minWatermark},\n        ]);\n      }\n      this.#watermark = minWatermark;\n    }\n  }\n\n  close() {\n    this.#downstream.cancel();\n  }\n}\n"],"mappings":";;;AAYA,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA,gCAAyB,IAAI,IAAyB;CAEtD,YACE,IACA,cACA,SACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKC,gBAAgB;EACrB,KAAKC,WAAW;CAClB;CAEA,KAAK,QAAyC;EAC5C,KAAK,MAAM,MAAM;EAEjB,MAAM,aAAa,CAAC;EACpB,MAAM,aAAa,aAAa,OAAwB,EACtD,eAAe,KAAKE,OAAO,QAAQ,UAAU,EAC/C,CAAC;EACD,KAAKD,cAAc,IAAI,QAAQ,IAAI,YAAY,YAAY,UAAU,CAAC;EACtE,KAAKH,IAAI,OAAO,oCAAoC,QAAQ;EAC5D,OAAO;CACT;CAEA,MAAM,QAAgB;EACpB,KAAKI,OAAO,QAAQ,KAAA,CAAS;CAC/B;CAEA,UAAU,QAAgB,QAA0C;EAClE,OAAO,KAAKD,cAAc,IAAI,MAAM,GAAG,KAAK,MAAM,KAAK;CACzD;CAEA,eAAe;EACb,OAAO,6BACL,KAAKF,cAAc,WACnB,KAAKA,cAAc,mBACnB,aACF;CACF;CAEA,OAAO,QAAgB,qBAA6C;EAClE,MAAM,MAAM,KAAKE,cAAc,IAAI,MAAM;EACzC,IACE,QACC,CAAC,uBAAuB,IAAI,eAAe,sBAC5C;GAEA,KAAKA,cAAc,OAAO,MAAM;GAChC,KAAKD,WAAW,MAAM;GACtB,IAAI,MAAM;GAEV,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ;GACpD,KAAKF,IAAI,OACP,kCAAkC,OAAO,IAAI,SAAS,KACxD;GAMA,sCAAsC,EAAE,SAAS,UAAU;IACzD,GAAG,KAAKK,aAAa;IACrB,QAAQ,sBAAsB,cAAc;IAC5C,WAAW,IAAI,UAAU;GAC3B,CAAC;EACH;CACF;CAEA,wBAAwB;EACtB,KAAK,MAAM,OAAO,KAAKF,cAAc,OAAO,GAC1C,IAAI,CAAC,IAAI,UAAU,GACjB,OAAO;EAGX,OAAO;CACT;CAEA,qBAA+B;EAC7B,OAAO,CAAC,GAAG,KAAKA,cAAc,QAAQ,CAAC,EACpC,QAAQ,GAAG,iBAAiB,CAAC,YAAY,UAAU,CAAC,EACpD,KAAK,CAAC,YAAY,MAAM;CAC7B;;CAGA,WACE,QACA,gBACA,cACA,QACM;EACN,MAAM,MAAM,KAAKA,cAAc,IAAI,MAAM;EACzC,IAAI,OAAO,CAAC,IAAI,UAAU,GAAG;GAC3B,KAAKH,IAAI,OACP,sCAAsC,aAAa,OAAO,QAC5D;GACA,IAAI,QAAQ,KAAKC,cAAc,WAAW,gBAAgB,YAAY;GAGtE,6CAA6C,EAAE,SAC7C,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ,GACnC;IAAC,GAAG,KAAKI,aAAa;IAAG;GAAM,CACjC;EACF;CACF;;;;;;CAOA,wBAAwB,QAAyB;EAC/C,OAAO,KAAKF,cAAc,IAAI,MAAM,GAAG,YAAY,KAAK;CAC1D;CAEA,wBAAwB;EACtB,OAAO,MAAM,KACX,KAAKA,cAAc,OAAO,IACzB,EAAC,wBAAuB,iBAC3B,EAAE,QAAO,cAAa,cAAc,IAAI;CAC1C;AACF;AAIA,IAAM,cAAN,MAAkB;CAChB;CACA,4BAA2B,IAAI,KAAK;CACpC;CACA,aAA4B;CAC5B,cAAc;CAEd,YACE,YACA,YACA;EACA,KAAK,aAAa;EAClB,KAAKG,cAAc;CACrB;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAKC;CACd;CAEA,YAAY;EACV,OAAO,KAAKA,eAAe;CAC7B;CAEA,KAAK,QAA0C;EAC7C,OAAO,KAAKD,gBAAgB;CAC9B;CAEA,cAAuB;EACrB,IAAI,KAAKE,aACP,OAAO;EAET,KAAKA,cAAc;EACnB,OAAO;CACT;CAEA,QAAQ,WAAmB,gBAAwB,cAAsB;EACvE,IAAI,KAAKD,eAAe,MAAM;GAC5B,IAAI,KAAKD,YAAY,QACnB,KAAKA,YAAY,KAAK,CACpB,UACA;IAAC,KAAK;IAAU;IAAW;IAAgB;GAAY,CACzD,CAAC;GAEH,KAAKC,aAAa;EACpB;CACF;CAEA,QAAQ;EACN,KAAKD,YAAY,OAAO;CAC1B;AACF"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts

@@ -24,6 +24,30 @@ export type SQLiteChangeLogCatchupOptions = {
     cleanupGuard?: SQLiteChangeLogCleanupGuard | undefined;
     sleep?: typeof sleep | undefined;
     now?: (() => number) | undefined;
+    /** Opens slice 11's process-local breaker after registration has committed. */
+    onFailure?: ((failure: SQLiteChangeLogCatchupFailure) => void) | undefined;
+};
+export type SQLiteChangeLogCatchupRequest = {
+    /**
+     * Whether the routing warm-up gate classified the log as warm *for this
+     * subscriber*. The coordinator is opened once and reused, so this cannot be
+     * a property of it: `coldReadPercent` serves some subscribers from a log
+     * that is still inside its warm-up window, and the catchup metrics have to
+     * separate those from steady-state serving.
+     */
+    readonly logWarm?: boolean | undefined;
+};
+export type SQLiteChangeLogCatchupFailure = 'barrier-timeout' | 'reader-error';
+export type SQLiteChangeLogCatchupRegistration = {
+    readonly kind: 'registered';
+} | {
+    readonly kind: 'uncovered';
+    readonly minWatermark: string;
+} | {
+    readonly kind: 'declined';
+    readonly error: unknown;
+} | {
+    readonly kind: 'handled';
 };
 /**
  * Coordinates a serving subscriber's gap-free transition from replica-local
@@ -42,7 +66,7 @@ export declare class SQLiteChangeLogCatchup implements Disposable {
      * the background so subscribe() does not wait for SQLite to reach the
      * required head.
      */
-    catchup(subscriber: Subscriber, captureRequiredHead: () => string | Promise<string>): Promise<void>;
+    catchup(subscriber: Subscriber, captureRequiredHead: () => string | Promise<string>, request?: SQLiteChangeLogCatchupRequest): Promise<SQLiteChangeLogCatchupRegistration>;
     /**
      * Notes that the change-log writer has committed `watermark`, i.e. that the
      * log's head has advanced to it. The writer runs in this process, so this is

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-catchup.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,KAAK,EAAC,MAAM,iCAAiC,CAAC;AAEtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE5D,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,+BAA+B,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,WAAW,4BAA4B;IAC3C,IAAI,CAAC,aAAa,EAAE,MAAM,GAAG,WAAW,CAAC;IACzC,IAAI,CACF,aAAa,EAAE,MAAM,EACrB,gBAAgB,EAAE,MAAM,EACxB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,WAAW,GACnB,aAAa,CAAC,SAAS,iBAAiB,EAAE,CAAC,CAAC;IAC/C,KAAK,IAAI,IAAI,CAAC;CACf;AAED;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IAC1C,oBAAoB,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CACxD;AAQD,MAAM,MAAM,6BAA6B,GAAG;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD,KAAK,CAAC,EAAE,OAAO,KAAK,GAAG,SAAS,CAAC;IACjC,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;CAClC,CAAC;AAMF;;;;;;;;GAQG;AACH,qBAAa,sBAAuB,YAAW,UAAU;;gBAiCrD,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,SAAS,EACpB,MAAM,EAAE,4BAA4B,EACpC,IAAI,EAAE,6BAA6B;IAcrC;;;;OAIG;IACG,OAAO,CACX,UAAU,EAAE,UAAU,EACtB,mBAAmB,EAAE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAClD,OAAO,CAAC,IAAI,CAAC;IA8BhB;;;;;;;;OAQG;IACH,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAS1C,MAAM,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IAMpC,KAAK,IAAI,IAAI;IAab,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAoRzB;AAID;;;;GAIG;AACH,qBAAa,2BAA4B,SAAQ,KAAK;CAAG;AAEzD,uEAAuE;AACvE,qBAAa,kCAAmC,SAAQ,2BAA2B;IACjF,QAAQ,CAAC,IAAI,wCAAwC;CACtD;AAED,iEAAiE;AACjE,qBAAa,kCAAmC,SAAQ,2BAA2B;IACjF,QAAQ,CAAC,IAAI,wCAAwC;CACtD"}
+{"version":3,"file":"sqlite-change-log-catchup.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAAC,KAAK,EAAC,MAAM,iCAAiC,CAAC;AAMtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE5D,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,+BAA+B,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,WAAW,4BAA4B;IAC3C,IAAI,CAAC,aAAa,EAAE,MAAM,GAAG,WAAW,CAAC;IACzC,IAAI,CACF,aAAa,EAAE,MAAM,EACrB,gBAAgB,EAAE,MAAM,EACxB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,WAAW,GACnB,aAAa,CAAC,SAAS,iBAAiB,EAAE,CAAC,CAAC;IAC/C,KAAK,IAAI,IAAI,CAAC;CACf;AAED;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IAC1C,oBAAoB,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CACxD;AAQD,MAAM,MAAM,6BAA6B,GAAG;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD,KAAK,CAAC,EAAE,OAAO,KAAK,GAAG,SAAS,CAAC;IACjC,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC,+EAA+E;IAC/E,SAAS,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,6BAA6B,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;CAC5E,CAAC;AAEF,MAAM,MAAM,6BAA6B,GAAG;IAC1C;;;;;;OAMG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACxC,CAAC;AAEF,MAAM,MAAM,6BAA6B,GAAG,iBAAiB,GAAG,cAAc,CAAC;AAE/E,MAAM,MAAM,kCAAkC,GAC1C;IAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAA;CAAC,GAC7B;IAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;CAAC,GAG3D;IAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;CAAC,GAEpD;IAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAA;CAAC,CAAC;AAM/B;;;;;;;;GAQG;AACH,qBAAa,sBAAuB,YAAW,UAAU;;gBA4ErD,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,SAAS,EACpB,MAAM,EAAE,4BAA4B,EACpC,IAAI,EAAE,6BAA6B;IAerC;;;;OAIG;IACG,OAAO,CACX,UAAU,EAAE,UAAU,EACtB,mBAAmB,EAAE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,EACnD,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,kCAAkC,CAAC;IAiE9C;;;;;;;;OAQG;IACH,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAS1C,MAAM,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IAMpC,KAAK,IAAI,IAAI;IAab,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAgUzB;AAID;;;;GAIG;AACH,qBAAa,2BAA4B,SAAQ,KAAK;CAAG;AAEzD,uEAAuE;AACvE,qBAAa,kCAAmC,SAAQ,2BAA2B;IACjF,QAAQ,CAAC,IAAI,wCAAwC;CACtD;AAED,iEAAiE;AACjE,qBAAa,kCAAmC,SAAQ,2BAA2B;IACjF,QAAQ,CAAC,IAAI,wCAAwC;CACtD"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js

@@ -1,6 +1,6 @@
 import { AbortError } from "../../../../shared/src/abort-error.js";
 import { sleep } from "../../../../shared/src/sleep.js";
-import { getOrCreateCounter } from "../../observability/metrics.js";
+import { getOrCreateCounter, getOrCreateLatencyHistogram, getOrCreateValueHistogram } from "../../observability/metrics.js";
 import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts
 var DEFAULT_BARRIER_POLL_INTERVAL_MS = 1e3;
@@ -24,9 +24,32 @@ var SQLiteChangeLogCatchup = class {
 	#cleanupGuard;
 	#sleep;
 	#now;
+	#onFailure;
 	#barrierTimeouts = getOrCreateCounter("replication", "sqlite_change_log.barrier_timeouts", "SQLite change-log catchups that timed out waiting for the required head.");
 	#barrierBacklogOverflows = getOrCreateCounter("replication", "sqlite_change_log.barrier_backlog_overflows", "SQLite change-log catchups abandoned because the subscriber backlog reached its high water mark while waiting for the required head.");
 	#barrierWakeups = getOrCreateCounter("replication", "sqlite_change_log.barrier_wakeups", "SQLite catchup barrier waits, labeled by what ended the wait. A population dominated by \"poll\" means the change-log writer's commit notification is not reaching the barrier.");
+	#catchupResults = getOrCreateCounter("replication", "sqlite_change_log.catchup_results", "SQLite catchup subscriptions by outcome.");
+	#catchupRows = getOrCreateCounter("replication", "sqlite_change_log.catchup_rows", "Rows served from the SQLite change log during catchup.");
+	#catchupBytes = getOrCreateCounter("replication", "sqlite_change_log.catchup_bytes", {
+		description: "Bytes served from the SQLite change log during catchup.",
+		unit: "By"
+	});
+	#catchupDuration = getOrCreateLatencyHistogram("replication", "sqlite_change_log.catchup_duration", "Duration of a SQLite change-log catchup.");
+	#barrierWaitDuration = getOrCreateLatencyHistogram("replication", "sqlite_change_log.barrier_wait_duration", "Time SQLite catchup waits for its pinned required head.");
+	#catchupBacklogPeak = getOrCreateValueHistogram("replication", "sqlite_change_log.catchup_backlog_peak_bytes", {
+		description: "Peak live backlog bytes buffered while a subscriber catches up from SQLite.",
+		unit: "By",
+		bucketBoundaries: [
+			0,
+			1024,
+			16384,
+			65536,
+			262144,
+			1048576,
+			4194304,
+			16777216
+		]
+	});
 	#catchups = /* @__PURE__ */ new Map();
 	#commitWaiters = /* @__PURE__ */ new Set();
 	#closed = false;
@@ -40,35 +63,58 @@ var SQLiteChangeLogCatchup = class {
 		this.#cleanupGuard = opts.cleanupGuard ?? NOOP_CLEANUP_GUARD;
 		this.#sleep = opts.sleep ?? sleep;
 		this.#now = opts.now ?? Date.now;
+		this.#onFailure = opts.onFailure;
 	}
 	/**
 	* Registers the subscriber before resolving. Catchup itself continues in
 	* the background so subscribe() does not wait for SQLite to reach the
 	* required head.
 	*/
-	async catchup(subscriber, captureRequiredHead) {
+	async catchup(subscriber, captureRequiredHead, request = {}) {
 		if (this.#closed) {
 			subscriber.fail(new AbortError("SQLite change-log catchup is closed"));
-			return;
+			return { kind: "handled" };
 		}
 		const abort = new AbortController();
 		this.#catchups.set(subscriber, abort);
 		let requiredHead;
+		let uncoveredMinWatermark;
+		let committed = false;
 		try {
 			await this.#cleanupGuard.runWhilePurgeBlocked(() => {
 				this.#throwIfAborted(abort.signal);
+				const plan = this.#reader.plan(subscriber.watermark);
+				if (plan.kind === "too-old") {
+					uncoveredMinWatermark = plan.minWatermark;
+					return;
+				}
 				requiredHead = captureRequiredHead();
 				this.#forwarder.add(subscriber);
+				committed = true;
 			});
 		} catch (error) {
 			this.#catchups.delete(subscriber);
-			if (!abort.signal.aborted) {
-				this.#lc.error?.(`error while registering SQLite catchup subscriber ${subscriber.id}`, error);
-				subscriber.fail(error);
+			if (abort.signal.aborted) return { kind: "handled" };
+			if (!committed) {
+				this.#onFailure?.("reader-error");
+				return {
+					kind: "declined",
+					error
+				};
 			}
-			return;
+			this.#lc.error?.(`error while registering SQLite catchup subscriber ${subscriber.id}`, error);
+			subscriber.fail(error);
+			return { kind: "handled" };
+		}
+		if (uncoveredMinWatermark !== void 0) {
+			this.#catchups.delete(subscriber);
+			return {
+				kind: "uncovered",
+				minWatermark: uncoveredMinWatermark
+			};
 		}
-		this.#run(subscriber, requiredHead, abort);
+		this.#run(subscriber, requiredHead, abort, request.logWarm);
+		return { kind: "registered" };
 	}
 	/**
 	* Notes that the change-log writer has committed `watermark`, i.e. that the
@@ -103,23 +149,34 @@ var SQLiteChangeLogCatchup = class {
 	[Symbol.dispose]() {
 		this.close();
 	}
-	async #run(subscriber, requiredHead, abort) {
+	async #run(subscriber, requiredHead, abort, logWarm) {
 		const { signal } = abort;
+		const catchupStart = this.#now();
+		let outcome = "aborted";
+		let backlogPeakBytes = 0;
 		try {
 			const requiredHeadStart = this.#now();
 			const required = await this.#awaitRequiredHead(requiredHead, signal);
 			const requiredHeadWaitMs = this.#now() - requiredHeadStart;
 			if (requiredHeadWaitMs > this.#barrierTimeoutMs) this.#lc.info?.(`waited ${requiredHeadWaitMs} ms for the forwarded transaction to commit before starting the SQLite barrier for ${subscriber.id}`);
 			const deadline = this.#now() + this.#barrierTimeoutMs;
-			const plan = await this.#waitForPlan(subscriber, required, deadline, signal);
+			const barrierStart = this.#now();
+			let plan;
+			try {
+				plan = await this.#waitForPlan(subscriber, required, deadline, signal);
+			} finally {
+				this.#barrierWaitDuration.recordMs(this.#now() - barrierStart);
+			}
 			this.#throwIfAborted(signal);
 			if (plan.kind === "too-old") {
+				outcome = "too-old";
 				const message = `earliest supported watermark is ${plan.minWatermark} (requested ${subscriber.watermark})`;
 				this.#lc.warn?.(`rejecting subscriber at watermark ${subscriber.watermark} (earliest watermark: ${plan.minWatermark})`);
 				subscriber.close(2, message);
 				return;
 			}
 			let count = 0;
+			let bytes = 0;
 			const start = this.#now();
 			if (plan.kind === "range") {
 				let lastBatchConsumed;
@@ -132,24 +189,50 @@ var SQLiteChangeLogCatchup = class {
 					for (const change of changes) {
 						lastBatchConsumed = subscriber.catchup(change);
 						count++;
+						bytes += Buffer.byteLength(change[2]);
 					}
+					backlogPeakBytes = Math.max(backlogPeakBytes, subscriber.getStats().backlogBytes);
 				}
 				await lastBatchConsumed;
-			} else this.#lc.warn?.(`subscriber ${subscriber.id} at watermark ${subscriber.watermark} is ahead of the SQLite change-log head ${plan.headWatermark}; waiting for the replica to catch up`);
+				outcome = "range";
+			} else {
+				outcome = "ahead";
+				this.#lc.warn?.(`subscriber ${subscriber.id} at watermark ${subscriber.watermark} is ahead of the SQLite change-log head ${plan.headWatermark}; waiting for the replica to catch up`);
+			}
 			this.#throwIfAborted(signal);
 			this.#lc.info?.(`caught up ${subscriber.id} from SQLite with ${count} changes (${this.#now() - start} ms)`);
+			const attributes = {
+				classification: outcome,
+				log_warm: logWarm ?? "unknown"
+			};
+			this.#catchupRows.add(count, attributes);
+			this.#catchupBytes.add(bytes, attributes);
 			subscriber.setCaughtUp();
 		} catch (error) {
 			if (signal.aborted) return;
 			if (error instanceof SQLiteChangeLogBarrierError) {
-				if (error instanceof SQLiteChangeLogBarrierTimeoutError) this.#barrierTimeouts.add(1);
+				if (error instanceof SQLiteChangeLogBarrierTimeoutError) {
+					this.#barrierTimeouts.add(1);
+					outcome = "barrier-timeout";
+					this.#onFailure?.("barrier-timeout");
+				} else outcome = "barrier-backlog";
 				this.#lc.warn?.(`ending subscription for ${subscriber.id} to retry SQLite catchup`, error);
 				subscriber.close();
 				return;
 			}
+			outcome = "reader-error";
+			this.#onFailure?.("reader-error");
 			this.#lc.error?.(`error while catching up subscriber ${subscriber.id} from SQLite`, error);
 			subscriber.fail(error);
 		} finally {
+			backlogPeakBytes = Math.max(backlogPeakBytes, subscriber.getStats().backlogBytes);
+			const attributes = {
+				classification: outcome,
+				log_warm: logWarm ?? "unknown"
+			};
+			this.#catchupResults.add(1, attributes);
+			this.#catchupDuration.recordMs(this.#now() - catchupStart, attributes);
+			this.#catchupBacklogPeak.record(backlogPeakBytes, attributes);
 			if (this.#catchups.get(subscriber) === abort) this.#catchups.delete(subscriber);
 		}
 	}