@rocicorp/zero 1.10.0-head-23da5f1c-20260822 → 1.10.0

package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js

@@ -2,6 +2,7 @@ import { assert } from "../../../../shared/src/asserts.js";
 import { must } from "../../../../shared/src/must.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
 import { RunningState, UnrecoverableError } from "../running-state.js";
+import { versionFromLexi } from "../../types/lexi-version.js";
 import { Subscription } from "../../types/subscription.js";
 import { litestreamBackupVerificationDuration, litestreamMonitorMetricAttrs } from "../litestream/metrics.js";
 import { statSync } from "node:fs";
@@ -160,11 +161,16 @@ var Litestream3PrometheusPoller = class {
 			const watermark = metric.labels?.watermark;
 			const name = metric.labels?.name;
 			const time = /* @__PURE__ */ new Date(parseFloat(metric.value) * 1e3);
-			if (watermark) yield {
-				watermark,
-				time,
-				name
-			};
+			if (watermark) try {
+				versionFromLexi(watermark);
+				yield {
+					watermark,
+					time,
+					name
+				};
+			} catch (e) {
+				this.#lc.warn?.(`ignoring invalid watermark ${watermark}`, { metric }, e);
+			}
 		}
 	}
 	async #checkWatermarks() {

package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map

@@ -1 +1 @@
-{"version":3,"file":"litestream3-prometheus-poller.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#state","#stream","#watermarks","#verifyBackupState","#purgesBlocked","#checkMetricsTimer","#runStartTime","#confirmRestorableBackup","#lastVerifiedUploadTime","#verifyBackupStateTimed","#checkWatermarks","#publishVerifiedWatermarks","#checkInitialBackupDeadline","#fetchWatermarks","#lastWatermark","#latestBackupTime","#maxBackupBefore","#confirmedDurable","#backupStaleSince","#shutDownOnWedgedBackup","#initialBackupGraceMs","#shutDownOnMissingInitialBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamBackupVerificationDuration,\n  litestreamMonitorMetricAttrs,\n} from '../litestream/metrics.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nexport const CHECK_INTERVAL_MS = 10_000;\n\n/**\n * Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n */\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/**\n * How long the actual backup state may remain *continuously* behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n *\n * A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes *some* such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n *\n * The change-log is conservatively *not* purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n */\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 * 60_000; // 15 minutes\n\nconst GiB = 1024 ** 3;\n\n/**\n * How long the BackupMonitor waits for the *first* restorable backup to appear\n * before concluding that the backup pipeline is broken and shutting the process\n * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).\n *\n * On a fresh stack the first backup is not restorable until the initial\n * Postgres->replica sync completes and litestream finishes uploading the\n * initial snapshot — a multipart upload that is not listable in the destination\n * until it is committed. The producer cannot observe that sub-snapshot progress,\n * so this is necessarily a generous timeout. It is scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance (~1 hour per\n * 50GB) so that it never fires before the platform would restart a view-syncer\n * that is waiting on the backup.\n */\nexport const INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 60_000; // 1 hour ...\nconst INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB; // ... per 50 GiB\n\n/**\n * How often the BackupMonitor re-checks whether the first restorable backup has\n * appeared while a view-syncer holds a snapshot reservation open during a cold\n * start.\n */\nexport const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10_000;\n\n/**\n * Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n *\n * See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n */\nexport type BackupStateVerifier = () => Promise<Date>;\n\n/**\n * The Litestream3PrometheusPoller polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and emits verified watermarks to a stream of {@link BackedUpWatermark}s.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Because the watermarks reported by litestream metrics reflect what\n * litestream *believes* has been backed up (which has been observed to\n * diverge from reality when uploads silently fail), the cleanup watermark is\n * only published after verifying it against the actual backup state in the\n * replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n */\nexport class Litestream3PrometheusPoller {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #state = new RunningState('litestream3-prometheus-poller');\n\n  readonly #stream: Subscription<BackedUpWatermark>;\n  readonly #watermarks = new Map<string, Date>();\n\n  readonly #verifyBackupState: BackupStateVerifier;\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup state ' +\n      '(as listed from the replica destination) could not be verified, or ' +\n      'is older than the backup progress claimed by litestream metrics. ' +\n      'A steadily increasing value indicates a wedged or failing backup. ' +\n      'The \"backup-wedged\" reason is emitted once just before the process ' +\n      'shuts itself down due to a persistently stale backup.',\n  });\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  // The time of the most recent object actually verified in the backup\n  // destination, or `null` if none has ever been verified. Only assigned from a\n  // successful verifyBackupState() and never cleared, so a non-null value also\n  // means a restorable backup has been confirmed to exist (which is what gates\n  // the snapshot `status` signal and satisfies the initial-backup deadline).\n  #lastVerifiedUploadTime: Date | null = null;\n  // Epoch ms at which the actual backup state was first observed to be\n  // continuously behind the watermark litestream claims to have backed up,\n  // or `null` while the backup is keeping up. Reset whenever a purge is\n  // confirmed against a real upload.\n  #backupStaleSince: number | null = null;\n  // Epoch ms at which run() started, used to bound how long to wait for the\n  // first restorable backup to appear. `null` until run() is called.\n  #runStartTime: number | null = null;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    verifyBackupState: BackupStateVerifier,\n  ) {\n    this.#lc = lc.withContext('component', 'litestream3-prometheus-poller');\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#verifyBackupState = verifyBackupState;\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        clearInterval(this.#checkMetricsTimer);\n        this.#state.stop(this.#lc);\n      },\n    });\n  }\n\n  /**\n   * Starts polling for watermarks and pushes new ones to returned stream.\n   * Must only be called once.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#runStartTime === null, `Already called start()`);\n    this.#runStartTime = Date.now();\n    this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint}`);\n    this.#checkMetricsTimer = setInterval(\n      this.checkVerifyAndPublishWatermarks,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#stream;\n  }\n\n  /**\n   * Verifies whether a restorable backup actually exists in the destination\n   * (as opposed to what litestream metrics merely claim). On success the\n   * verified upload time is cached — seeding the cleanup-verification fast path\n   * ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.\n   * Returns false if no backup is listable yet (e.g. the initial snapshot is\n   * still uploading).\n   */\n  async #confirmRestorableBackup(): Promise<boolean> {\n    try {\n      this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      return true;\n    } catch (e) {\n      this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);\n      return false;\n    }\n  }\n\n  // Exported for testing\n  readonly checkVerifyAndPublishWatermarks = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      await this.#publishVerifiedWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n    try {\n      await this.#checkInitialBackupDeadline();\n    } catch (e) {\n      this.#lc.warn?.(`error checking initial backup deadline`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  async #publishVerifiedWatermarks() {\n    const maxBackup = this.#maxBackupBefore(Date.now());\n    if (!maxBackup) {\n      return;\n    }\n    const {watermark: maxWatermark, backupTime: claimedTime} = maxBackup;\n    // Purge guard: the watermarks (and their backup times) come from\n    // litestream metrics, which are exported when litestream *believes*\n    // an upload succeeded, and have been observed to advance even when\n    // nothing was actually written to the backup destination. Purging the\n    // change-log based on a falsely advancing watermark permanently breaks\n    // the ability to restore + catch up. Before advancing the cleanup\n    // watermark, verify it against the actual backup state: a claimed\n    // backup time is only trusted if an object was actually uploaded to\n    // the replica destination at (or after) that time, modulo clock skew.\n    if (!this.#confirmedDurable(claimedTime)) {\n      try {\n        this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      } catch (e) {\n        this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n        // Skipping the purge is safe: the change-log just grows.\n        this.#lc.warn?.(\n          `unable to verify backup state. skipping change-log cleanup ` +\n            `up to watermark ${maxWatermark} ` +\n            `(claimed backup time ${claimedTime.toISOString()})`,\n          e,\n        );\n        return;\n      }\n    }\n    // Watermarks whose backup time isn't yet confirmed durable remain in the\n    // map and are re-evaluated at the next check.\n    const lastUpload = must(this.#lastVerifiedUploadTime);\n    const verifiedBackup = this.#maxBackupBefore(\n      lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n    );\n    if (!verifiedBackup) {\n      this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n      const now = Date.now();\n      if (this.#backupStaleSince === null) {\n        this.#backupStaleSince = now;\n      }\n      const staleForMs = now - this.#backupStaleSince;\n      this.#lc.warn?.(\n        `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n          `litestream claims it was backed up at ` +\n          `${claimedTime.toISOString()}, but the last object actually ` +\n          `uploaded to ${this.#backupURL} was at ` +\n          `${lastUpload.toISOString()}. ` +\n          `The backup has been stale for ${staleForMs} ms.`,\n      );\n      if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n        this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n      }\n      return;\n    }\n    // The cleanup watermark advanced past a real upload, so the backup is\n    // keeping up: clear the staleness clock.\n    this.#backupStaleSince = null;\n    const {watermark: verifiedWatermark, backupTime} = verifiedBackup;\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= verifiedWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#stream.push({\n      watermark: verifiedWatermark,\n      backupTimeMs: backupTime.getTime(),\n    });\n    this.#lastWatermark = verifiedWatermark;\n  }\n\n  async #verifyBackupStateTimed(): Promise<Date> {\n    const start = performance.now();\n    let result: 'success' | 'error' = 'error';\n    try {\n      const ret = await this.#verifyBackupState();\n      result = 'success';\n      return ret;\n    } finally {\n      litestreamBackupVerificationDuration().recordMs(\n        performance.now() - start,\n        {\n          ...litestreamMonitorMetricAttrs(\n            this.#backupURL,\n            'legacy',\n            'replication_manager',\n          ),\n          result,\n        },\n      );\n    }\n  }\n\n  /**\n   * Called when the backup has been continuously stale for longer than\n   * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n   * backup, so the process is shut down by rejecting the {@link run()}\n   * promise. The exit is non-zero and logged at ERROR for alerting; on\n   * restart the monitor re-verifies, so a still-wedged backup keeps the\n   * process down until the destination is fixed.\n   */\n  #shutDownOnWedgedBackup(\n    staleForMs: number,\n    claimedTime: Date,\n    lastUpload: Date,\n  ) {\n    this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n    const err = new UnrecoverableError(\n      `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n        `at ${claimedTime.toISOString()}, but the last object actually ` +\n        `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n        `not advanced for ${staleForMs} ms (grace period ` +\n        `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n        `the backup; investigate the backup destination.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * On a fresh stack, fails loudly if no restorable backup has appeared within\n   * a generous, replica-size-scaled grace period (see\n   * {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the \"give up\" decision\n   * onto the replication-manager — the producer responsible for creating the\n   * backup — instead of leaving view-syncers to wait (and the platform to keep\n   * restarting them) while the real fault is here. Does nothing once a\n   * restorable backup has been confirmed.\n   */\n  async #checkInitialBackupDeadline() {\n    if (this.#lastVerifiedUploadTime !== null || this.#runStartTime === null) {\n      return;\n    }\n    const elapsed = Date.now() - this.#runStartTime;\n    if (elapsed < this.#initialBackupGraceMs()) {\n      return;\n    }\n    // The deadline has elapsed without a confirmed backup. Do one definitive\n    // check against the destination before giving up, in case a backup landed\n    // but was never exercised by a reservation or a cleanup verification.\n    if (await this.#confirmRestorableBackup()) {\n      return;\n    }\n    this.#shutDownOnMissingInitialBackup(elapsed);\n  }\n\n  /**\n   * The grace period for the first restorable backup, scaled by replica size to\n   * mirror the platform's storage-scaled startup-probe allowance. Falls back to\n   * the minimum (one unit) if the replica file size cannot be determined.\n   */\n  #initialBackupGraceMs(): number {\n    let bytes = 0;\n    try {\n      bytes = statSync(this.#replicaFile).size;\n    } catch {\n      // Replica file not present/readable yet; fall back to the minimum grace.\n    }\n    const units = Math.max(\n      1,\n      Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES),\n    );\n    return units * INITIAL_BACKUP_GRACE_MS_PER_UNIT;\n  }\n\n  /**\n   * Called when no restorable backup has appeared within\n   * {@link #initialBackupGraceMs}. Continuing to run is pointless — no\n   * view-syncer can restore — and the fault is the backup pipeline, so the\n   * process exits non-zero with a logged error for alerting, mirroring\n   * {@link #shutDownOnWedgedBackup}.\n   */\n  #shutDownOnMissingInitialBackup(elapsedMs: number) {\n    const err = new UnrecoverableError(\n      `no restorable backup has appeared at ${this.#backupURL} within ` +\n        `${elapsedMs} ms of startup (grace period ` +\n        `${this.#initialBackupGraceMs()} ms). The initial backup pipeline ` +\n        `appears to be broken; shutting down so that the replication-manager ` +\n        `is flagged as the cause rather than the view-syncers waiting on it. ` +\n        `Investigate litestream replication to ${this.#backupURL}.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * Returns the newest watermark whose backup time is at or before `cutoff`\n   * (epoch ms), or `''` if there is none.\n   */\n  #maxBackupBefore(\n    cutoff: number,\n  ): {watermark: string; backupTime: Date} | null {\n    let max = '';\n    let maxBackupTime: Date | undefined;\n    for (const [watermark, backupTime] of this.#watermarks) {\n      if (backupTime.getTime() <= cutoff && watermark > max) {\n        max = watermark;\n        maxBackupTime = backupTime;\n      }\n    }\n    if (!maxBackupTime) {\n      return null;\n    }\n    return {watermark: max, backupTime: maxBackupTime};\n  }\n\n  /**\n   * Returns `true` if the actual backup state, as last verified against the\n   * backup destination, confirms that data claimed to be backed up at\n   * `claimedTime` is durable (i.e. an object was actually uploaded at or\n   * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n   * clock skew).\n   */\n  #confirmedDurable(claimedTime: Date): boolean {\n    const lastUpload = this.#lastVerifiedUploadTime;\n    return (\n      lastUpload !== null &&\n      claimedTime.getTime() <=\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n    );\n  }\n}\n"],"mappings":";;;;;;;;;AAeA,IAAa,oBAAoB;;;;;AAMjC,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B,KAAK;AAE7C,IAAM,MAAM,QAAQ;;;;;;;;;;;;;;;AAgBpB,IAAa,mCAAmC,KAAK;AACrD,IAAM,kCAAkC,KAAK;;;;;;;;;;;;;;;;;AAmC7C,IAAa,8BAAb,MAAyC;CACvC;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,+BAA+B;CAElE;CACA,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAED,iBAAyB;CACzB,oBAAiC;CAMjC,0BAAuC;CAKvC,oBAAmC;CAGnC,gBAA+B;CAC/B;CAEA,YACE,IACA,aACA,WACA,iBACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,+BAA+B;EACtE,KAAKC,eAAe;EACpB,KAAKC,aAAa;EAClB,KAAKC,mBAAmB;EACxB,KAAKI,qBAAqB;EAC1B,KAAKF,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAKI,kBAAkB;GACrC,KAAKL,OAAO,KAAK,KAAKJ,GAAG;EAC3B,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKU,kBAAkB,MAAM,wBAAwB;EAC5D,KAAKA,gBAAgB,KAAK,IAAI;EAC9B,KAAKV,IAAI,OAAO,yBAAyB,KAAKG,kBAAkB;EAChE,KAAKM,qBAAqB,YACxB,KAAK,iCACL,iBACF;EACA,OAAO,KAAKJ;CACd;;;;;;;;;CAUA,MAAMM,2BAA6C;EACjD,IAAI;GACF,KAAKC,0BAA0B,MAAM,KAAKC,wBAAwB;GAClE,OAAO;EACT,SAAS,GAAG;GACV,KAAKb,IAAI,OAAO,gCAAgC,KAAKE,cAAc,CAAC;GACpE,OAAO;EACT;CACF;CAGA,kCAA2C,YAAY;EACrD,IAAI;GACF,MAAM,KAAKY,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKd,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAKY,2BAA2B;EACxC,SAAS,GAAG;GACV,KAAKf,IAAI,OAAO,4BAA4B,CAAC;EAC/C;EACA,IAAI;GACF,MAAM,KAAKgB,4BAA4B;EACzC,SAAS,GAAG;GACV,KAAKhB,IAAI,OAAO,0CAA0C,CAAC;EAC7D;CACF;CAEA,OAAOiB,mBAIJ;EACD,MAAM,kBAAkB,KAAKd;EAC7B,MAAM,SAAS,KAAKC,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAKJ,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAKH,IAAI,OACP,8BAA8B,KAAKG,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GAErD,IAAI,WACF,MAAM;IAAC;IAAW;IAAM;GAAI;EAEhC;CAGN;CAEA,MAAMW,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAKG,iBAAiB,GAChE,IAAI,YAAY,KAAKC,kBAAkB,CAAC,KAAKZ,YAAY,IAAI,SAAS,GAAG;GACvE,KAAKN,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAKM,YAAY,IAAI,WAAW,IAAI;GACpC,KAAKa,oBAAoB;EAC3B;EAEF,OAAO,KAAKA;CACd;CAEA,MAAMJ,6BAA6B;EACjC,MAAM,YAAY,KAAKK,iBAAiB,KAAK,IAAI,CAAC;EAClD,IAAI,CAAC,WACH;EAEF,MAAM,EAAC,WAAW,cAAc,YAAY,gBAAe;EAU3D,IAAI,CAAC,KAAKC,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAKT,0BAA0B,MAAM,KAAKC,wBAAwB;EACpE,SAAS,GAAG;GACV,KAAKL,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAKR,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAKY,uBAAuB;EACpD,MAAM,iBAAiB,KAAKQ,iBAC1B,WAAW,QAAQ,IAAI,4BACzB;EACA,IAAI,CAAC,gBAAgB;GACnB,KAAKZ,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAKc,sBAAsB,MAC7B,KAAKA,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAKA;GAC9B,KAAKtB,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAKE,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAKqB,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAKD,oBAAoB;EACzB,MAAM,EAAC,WAAW,mBAAmB,eAAc;EACnD,KAAK,MAAM,aAAa,KAAKhB,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKD,QAAQ,KAAK;GAChB,WAAW;GACX,cAAc,WAAW,QAAQ;EACnC,CAAC;EACD,KAAKa,iBAAiB;CACxB;CAEA,MAAML,0BAAyC;EAC7C,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAKN,mBAAmB;GAC1C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,EAAE,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAKL,YACL,UACA,qBACF;IACA;GACF,CACF;EACF;CACF;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAKM,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAKN,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;;;;;;CAWA,MAAMW,8BAA8B;EAClC,IAAI,KAAKJ,4BAA4B,QAAQ,KAAKF,kBAAkB,MAClE;EAEF,MAAM,UAAU,KAAK,IAAI,IAAI,KAAKA;EAClC,IAAI,UAAU,KAAKc,sBAAsB,GACvC;EAKF,IAAI,MAAM,KAAKb,yBAAyB,GACtC;EAEF,KAAKc,gCAAgC,OAAO;CAC9C;;;;;;CAOA,wBAAgC;EAC9B,IAAI,QAAQ;EACZ,IAAI;GACF,QAAQ,SAAS,KAAKxB,YAAY,EAAE;EACtC,QAAQ,CAER;EAKA,OAJc,KAAK,IACjB,GACA,KAAK,KAAK,QAAQ,+BAA+B,CAE5C,IAAQ;CACjB;;;;;;;;CASA,gCAAgC,WAAmB;EACjD,MAAM,MAAM,IAAI,mBACd,wCAAwC,KAAKC,WAAW,UACnD,UAAU,+BACV,KAAKsB,sBAAsB,EAAE,kNAGS,KAAKtB,WAAW,EAC7D;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;CAMA,iBACE,QAC8C;EAC9C,IAAI,MAAM;EACV,IAAI;EACJ,KAAK,MAAM,CAAC,WAAW,eAAe,KAAKC,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAAK;GACrD,MAAM;GACN,gBAAgB;EAClB;EAEF,IAAI,CAAC,eACH,OAAO;EAET,OAAO;GAAC,WAAW;GAAK,YAAY;EAAa;CACnD;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAKM;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;AACF"}
+{"version":3,"file":"litestream3-prometheus-poller.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#state","#stream","#watermarks","#verifyBackupState","#purgesBlocked","#checkMetricsTimer","#runStartTime","#confirmRestorableBackup","#lastVerifiedUploadTime","#verifyBackupStateTimed","#checkWatermarks","#publishVerifiedWatermarks","#checkInitialBackupDeadline","#fetchWatermarks","#lastWatermark","#latestBackupTime","#maxBackupBefore","#confirmedDurable","#backupStaleSince","#shutDownOnWedgedBackup","#initialBackupGraceMs","#shutDownOnMissingInitialBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamBackupVerificationDuration,\n  litestreamMonitorMetricAttrs,\n} from '../litestream/metrics.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nexport const CHECK_INTERVAL_MS = 10_000;\n\n/**\n * Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n */\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/**\n * How long the actual backup state may remain *continuously* behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n *\n * A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes *some* such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n *\n * The change-log is conservatively *not* purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n */\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 * 60_000; // 15 minutes\n\nconst GiB = 1024 ** 3;\n\n/**\n * How long the BackupMonitor waits for the *first* restorable backup to appear\n * before concluding that the backup pipeline is broken and shutting the process\n * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).\n *\n * On a fresh stack the first backup is not restorable until the initial\n * Postgres->replica sync completes and litestream finishes uploading the\n * initial snapshot — a multipart upload that is not listable in the destination\n * until it is committed. The producer cannot observe that sub-snapshot progress,\n * so this is necessarily a generous timeout. It is scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance (~1 hour per\n * 50GB) so that it never fires before the platform would restart a view-syncer\n * that is waiting on the backup.\n */\nexport const INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 60_000; // 1 hour ...\nconst INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB; // ... per 50 GiB\n\n/**\n * How often the BackupMonitor re-checks whether the first restorable backup has\n * appeared while a view-syncer holds a snapshot reservation open during a cold\n * start.\n */\nexport const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10_000;\n\n/**\n * Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n *\n * See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n */\nexport type BackupStateVerifier = () => Promise<Date>;\n\n/**\n * The Litestream3PrometheusPoller polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and emits verified watermarks to a stream of {@link BackedUpWatermark}s.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Because the watermarks reported by litestream metrics reflect what\n * litestream *believes* has been backed up (which has been observed to\n * diverge from reality when uploads silently fail), the cleanup watermark is\n * only published after verifying it against the actual backup state in the\n * replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n */\nexport class Litestream3PrometheusPoller {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #state = new RunningState('litestream3-prometheus-poller');\n\n  readonly #stream: Subscription<BackedUpWatermark>;\n  readonly #watermarks = new Map<string, Date>();\n\n  readonly #verifyBackupState: BackupStateVerifier;\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup state ' +\n      '(as listed from the replica destination) could not be verified, or ' +\n      'is older than the backup progress claimed by litestream metrics. ' +\n      'A steadily increasing value indicates a wedged or failing backup. ' +\n      'The \"backup-wedged\" reason is emitted once just before the process ' +\n      'shuts itself down due to a persistently stale backup.',\n  });\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  // The time of the most recent object actually verified in the backup\n  // destination, or `null` if none has ever been verified. Only assigned from a\n  // successful verifyBackupState() and never cleared, so a non-null value also\n  // means a restorable backup has been confirmed to exist (which is what gates\n  // the snapshot `status` signal and satisfies the initial-backup deadline).\n  #lastVerifiedUploadTime: Date | null = null;\n  // Epoch ms at which the actual backup state was first observed to be\n  // continuously behind the watermark litestream claims to have backed up,\n  // or `null` while the backup is keeping up. Reset whenever a purge is\n  // confirmed against a real upload.\n  #backupStaleSince: number | null = null;\n  // Epoch ms at which run() started, used to bound how long to wait for the\n  // first restorable backup to appear. `null` until run() is called.\n  #runStartTime: number | null = null;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    verifyBackupState: BackupStateVerifier,\n  ) {\n    this.#lc = lc.withContext('component', 'litestream3-prometheus-poller');\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#verifyBackupState = verifyBackupState;\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        clearInterval(this.#checkMetricsTimer);\n        this.#state.stop(this.#lc);\n      },\n    });\n  }\n\n  /**\n   * Starts polling for watermarks and pushes new ones to returned stream.\n   * Must only be called once.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#runStartTime === null, `Already called start()`);\n    this.#runStartTime = Date.now();\n    this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint}`);\n    this.#checkMetricsTimer = setInterval(\n      this.checkVerifyAndPublishWatermarks,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#stream;\n  }\n\n  /**\n   * Verifies whether a restorable backup actually exists in the destination\n   * (as opposed to what litestream metrics merely claim). On success the\n   * verified upload time is cached — seeding the cleanup-verification fast path\n   * ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.\n   * Returns false if no backup is listable yet (e.g. the initial snapshot is\n   * still uploading).\n   */\n  async #confirmRestorableBackup(): Promise<boolean> {\n    try {\n      this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      return true;\n    } catch (e) {\n      this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);\n      return false;\n    }\n  }\n\n  // Exported for testing\n  readonly checkVerifyAndPublishWatermarks = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      await this.#publishVerifiedWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n    try {\n      await this.#checkInitialBackupDeadline();\n    } catch (e) {\n      this.#lc.warn?.(`error checking initial backup deadline`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n          if (watermark) {\n            try {\n              versionFromLexi(watermark); // Sanity check: Ensure that it is valid.\n              yield {watermark, time, name};\n            } catch (e) {\n              this.#lc.warn?.(\n                `ignoring invalid watermark ${watermark}`,\n                {metric},\n                e,\n              );\n            }\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  async #publishVerifiedWatermarks() {\n    const maxBackup = this.#maxBackupBefore(Date.now());\n    if (!maxBackup) {\n      return;\n    }\n    const {watermark: maxWatermark, backupTime: claimedTime} = maxBackup;\n    // Purge guard: the watermarks (and their backup times) come from\n    // litestream metrics, which are exported when litestream *believes*\n    // an upload succeeded, and have been observed to advance even when\n    // nothing was actually written to the backup destination. Purging the\n    // change-log based on a falsely advancing watermark permanently breaks\n    // the ability to restore + catch up. Before advancing the cleanup\n    // watermark, verify it against the actual backup state: a claimed\n    // backup time is only trusted if an object was actually uploaded to\n    // the replica destination at (or after) that time, modulo clock skew.\n    if (!this.#confirmedDurable(claimedTime)) {\n      try {\n        this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      } catch (e) {\n        this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n        // Skipping the purge is safe: the change-log just grows.\n        this.#lc.warn?.(\n          `unable to verify backup state. skipping change-log cleanup ` +\n            `up to watermark ${maxWatermark} ` +\n            `(claimed backup time ${claimedTime.toISOString()})`,\n          e,\n        );\n        return;\n      }\n    }\n    // Watermarks whose backup time isn't yet confirmed durable remain in the\n    // map and are re-evaluated at the next check.\n    const lastUpload = must(this.#lastVerifiedUploadTime);\n    const verifiedBackup = this.#maxBackupBefore(\n      lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n    );\n    if (!verifiedBackup) {\n      this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n      const now = Date.now();\n      if (this.#backupStaleSince === null) {\n        this.#backupStaleSince = now;\n      }\n      const staleForMs = now - this.#backupStaleSince;\n      this.#lc.warn?.(\n        `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n          `litestream claims it was backed up at ` +\n          `${claimedTime.toISOString()}, but the last object actually ` +\n          `uploaded to ${this.#backupURL} was at ` +\n          `${lastUpload.toISOString()}. ` +\n          `The backup has been stale for ${staleForMs} ms.`,\n      );\n      if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n        this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n      }\n      return;\n    }\n    // The cleanup watermark advanced past a real upload, so the backup is\n    // keeping up: clear the staleness clock.\n    this.#backupStaleSince = null;\n    const {watermark: verifiedWatermark, backupTime} = verifiedBackup;\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= verifiedWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#stream.push({\n      watermark: verifiedWatermark,\n      backupTimeMs: backupTime.getTime(),\n    });\n    this.#lastWatermark = verifiedWatermark;\n  }\n\n  async #verifyBackupStateTimed(): Promise<Date> {\n    const start = performance.now();\n    let result: 'success' | 'error' = 'error';\n    try {\n      const ret = await this.#verifyBackupState();\n      result = 'success';\n      return ret;\n    } finally {\n      litestreamBackupVerificationDuration().recordMs(\n        performance.now() - start,\n        {\n          ...litestreamMonitorMetricAttrs(\n            this.#backupURL,\n            'legacy',\n            'replication_manager',\n          ),\n          result,\n        },\n      );\n    }\n  }\n\n  /**\n   * Called when the backup has been continuously stale for longer than\n   * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n   * backup, so the process is shut down by rejecting the {@link run()}\n   * promise. The exit is non-zero and logged at ERROR for alerting; on\n   * restart the monitor re-verifies, so a still-wedged backup keeps the\n   * process down until the destination is fixed.\n   */\n  #shutDownOnWedgedBackup(\n    staleForMs: number,\n    claimedTime: Date,\n    lastUpload: Date,\n  ) {\n    this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n    const err = new UnrecoverableError(\n      `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n        `at ${claimedTime.toISOString()}, but the last object actually ` +\n        `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n        `not advanced for ${staleForMs} ms (grace period ` +\n        `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n        `the backup; investigate the backup destination.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * On a fresh stack, fails loudly if no restorable backup has appeared within\n   * a generous, replica-size-scaled grace period (see\n   * {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the \"give up\" decision\n   * onto the replication-manager — the producer responsible for creating the\n   * backup — instead of leaving view-syncers to wait (and the platform to keep\n   * restarting them) while the real fault is here. Does nothing once a\n   * restorable backup has been confirmed.\n   */\n  async #checkInitialBackupDeadline() {\n    if (this.#lastVerifiedUploadTime !== null || this.#runStartTime === null) {\n      return;\n    }\n    const elapsed = Date.now() - this.#runStartTime;\n    if (elapsed < this.#initialBackupGraceMs()) {\n      return;\n    }\n    // The deadline has elapsed without a confirmed backup. Do one definitive\n    // check against the destination before giving up, in case a backup landed\n    // but was never exercised by a reservation or a cleanup verification.\n    if (await this.#confirmRestorableBackup()) {\n      return;\n    }\n    this.#shutDownOnMissingInitialBackup(elapsed);\n  }\n\n  /**\n   * The grace period for the first restorable backup, scaled by replica size to\n   * mirror the platform's storage-scaled startup-probe allowance. Falls back to\n   * the minimum (one unit) if the replica file size cannot be determined.\n   */\n  #initialBackupGraceMs(): number {\n    let bytes = 0;\n    try {\n      bytes = statSync(this.#replicaFile).size;\n    } catch {\n      // Replica file not present/readable yet; fall back to the minimum grace.\n    }\n    const units = Math.max(\n      1,\n      Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES),\n    );\n    return units * INITIAL_BACKUP_GRACE_MS_PER_UNIT;\n  }\n\n  /**\n   * Called when no restorable backup has appeared within\n   * {@link #initialBackupGraceMs}. Continuing to run is pointless — no\n   * view-syncer can restore — and the fault is the backup pipeline, so the\n   * process exits non-zero with a logged error for alerting, mirroring\n   * {@link #shutDownOnWedgedBackup}.\n   */\n  #shutDownOnMissingInitialBackup(elapsedMs: number) {\n    const err = new UnrecoverableError(\n      `no restorable backup has appeared at ${this.#backupURL} within ` +\n        `${elapsedMs} ms of startup (grace period ` +\n        `${this.#initialBackupGraceMs()} ms). The initial backup pipeline ` +\n        `appears to be broken; shutting down so that the replication-manager ` +\n        `is flagged as the cause rather than the view-syncers waiting on it. ` +\n        `Investigate litestream replication to ${this.#backupURL}.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * Returns the newest watermark whose backup time is at or before `cutoff`\n   * (epoch ms), or `''` if there is none.\n   */\n  #maxBackupBefore(\n    cutoff: number,\n  ): {watermark: string; backupTime: Date} | null {\n    let max = '';\n    let maxBackupTime: Date | undefined;\n    for (const [watermark, backupTime] of this.#watermarks) {\n      if (backupTime.getTime() <= cutoff && watermark > max) {\n        max = watermark;\n        maxBackupTime = backupTime;\n      }\n    }\n    if (!maxBackupTime) {\n      return null;\n    }\n    return {watermark: max, backupTime: maxBackupTime};\n  }\n\n  /**\n   * Returns `true` if the actual backup state, as last verified against the\n   * backup destination, confirms that data claimed to be backed up at\n   * `claimedTime` is durable (i.e. an object was actually uploaded at or\n   * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n   * clock skew).\n   */\n  #confirmedDurable(claimedTime: Date): boolean {\n    const lastUpload = this.#lastVerifiedUploadTime;\n    return (\n      lastUpload !== null &&\n      claimedTime.getTime() <=\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n    );\n  }\n}\n"],"mappings":";;;;;;;;;;AAgBA,IAAa,oBAAoB;;;;;AAMjC,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B,KAAK;AAE7C,IAAM,MAAM,QAAQ;;;;;;;;;;;;;;;AAgBpB,IAAa,mCAAmC,KAAK;AACrD,IAAM,kCAAkC,KAAK;;;;;;;;;;;;;;;;;AAmC7C,IAAa,8BAAb,MAAyC;CACvC;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,+BAA+B;CAElE;CACA,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAED,iBAAyB;CACzB,oBAAiC;CAMjC,0BAAuC;CAKvC,oBAAmC;CAGnC,gBAA+B;CAC/B;CAEA,YACE,IACA,aACA,WACA,iBACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,+BAA+B;EACtE,KAAKC,eAAe;EACpB,KAAKC,aAAa;EAClB,KAAKC,mBAAmB;EACxB,KAAKI,qBAAqB;EAC1B,KAAKF,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAKI,kBAAkB;GACrC,KAAKL,OAAO,KAAK,KAAKJ,GAAG;EAC3B,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKU,kBAAkB,MAAM,wBAAwB;EAC5D,KAAKA,gBAAgB,KAAK,IAAI;EAC9B,KAAKV,IAAI,OAAO,yBAAyB,KAAKG,kBAAkB;EAChE,KAAKM,qBAAqB,YACxB,KAAK,iCACL,iBACF;EACA,OAAO,KAAKJ;CACd;;;;;;;;;CAUA,MAAMM,2BAA6C;EACjD,IAAI;GACF,KAAKC,0BAA0B,MAAM,KAAKC,wBAAwB;GAClE,OAAO;EACT,SAAS,GAAG;GACV,KAAKb,IAAI,OAAO,gCAAgC,KAAKE,cAAc,CAAC;GACpE,OAAO;EACT;CACF;CAGA,kCAA2C,YAAY;EACrD,IAAI;GACF,MAAM,KAAKY,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKd,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAKY,2BAA2B;EACxC,SAAS,GAAG;GACV,KAAKf,IAAI,OAAO,4BAA4B,CAAC;EAC/C;EACA,IAAI;GACF,MAAM,KAAKgB,4BAA4B;EACzC,SAAS,GAAG;GACV,KAAKhB,IAAI,OAAO,0CAA0C,CAAC;EAC7D;CACF;CAEA,OAAOiB,mBAIJ;EACD,MAAM,kBAAkB,KAAKd;EAC7B,MAAM,SAAS,KAAKC,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAKJ,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAKH,IAAI,OACP,8BAA8B,KAAKG,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GACrD,IAAI,WACF,IAAI;IACF,gBAAgB,SAAS;IACzB,MAAM;KAAC;KAAW;KAAM;IAAI;GAC9B,SAAS,GAAG;IACV,KAAKH,IAAI,OACP,8BAA8B,aAC9B,EAAC,OAAM,GACP,CACF;GACF;EAEJ;CAGN;CAEA,MAAMc,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAKG,iBAAiB,GAChE,IAAI,YAAY,KAAKC,kBAAkB,CAAC,KAAKZ,YAAY,IAAI,SAAS,GAAG;GACvE,KAAKN,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAKM,YAAY,IAAI,WAAW,IAAI;GACpC,KAAKa,oBAAoB;EAC3B;EAEF,OAAO,KAAKA;CACd;CAEA,MAAMJ,6BAA6B;EACjC,MAAM,YAAY,KAAKK,iBAAiB,KAAK,IAAI,CAAC;EAClD,IAAI,CAAC,WACH;EAEF,MAAM,EAAC,WAAW,cAAc,YAAY,gBAAe;EAU3D,IAAI,CAAC,KAAKC,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAKT,0BAA0B,MAAM,KAAKC,wBAAwB;EACpE,SAAS,GAAG;GACV,KAAKL,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAKR,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAKY,uBAAuB;EACpD,MAAM,iBAAiB,KAAKQ,iBAC1B,WAAW,QAAQ,IAAI,4BACzB;EACA,IAAI,CAAC,gBAAgB;GACnB,KAAKZ,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAKc,sBAAsB,MAC7B,KAAKA,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAKA;GAC9B,KAAKtB,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAKE,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAKqB,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAKD,oBAAoB;EACzB,MAAM,EAAC,WAAW,mBAAmB,eAAc;EACnD,KAAK,MAAM,aAAa,KAAKhB,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKD,QAAQ,KAAK;GAChB,WAAW;GACX,cAAc,WAAW,QAAQ;EACnC,CAAC;EACD,KAAKa,iBAAiB;CACxB;CAEA,MAAML,0BAAyC;EAC7C,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAKN,mBAAmB;GAC1C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,EAAE,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAKL,YACL,UACA,qBACF;IACA;GACF,CACF;EACF;CACF;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAKM,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAKN,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;;;;;;CAWA,MAAMW,8BAA8B;EAClC,IAAI,KAAKJ,4BAA4B,QAAQ,KAAKF,kBAAkB,MAClE;EAEF,MAAM,UAAU,KAAK,IAAI,IAAI,KAAKA;EAClC,IAAI,UAAU,KAAKc,sBAAsB,GACvC;EAKF,IAAI,MAAM,KAAKb,yBAAyB,GACtC;EAEF,KAAKc,gCAAgC,OAAO;CAC9C;;;;;;CAOA,wBAAgC;EAC9B,IAAI,QAAQ;EACZ,IAAI;GACF,QAAQ,SAAS,KAAKxB,YAAY,EAAE;EACtC,QAAQ,CAER;EAKA,OAJc,KAAK,IACjB,GACA,KAAK,KAAK,QAAQ,+BAA+B,CAE5C,IAAQ;CACjB;;;;;;;;CASA,gCAAgC,WAAmB;EACjD,MAAM,MAAM,IAAI,mBACd,wCAAwC,KAAKC,WAAW,UACnD,UAAU,+BACV,KAAKsB,sBAAsB,EAAE,kNAGS,KAAKtB,WAAW,EAC7D;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;CAMA,iBACE,QAC8C;EAC9C,IAAI,MAAM;EACV,IAAI;EACJ,KAAK,MAAM,CAAC,WAAW,eAAe,KAAKC,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAAK;GACrD,MAAM;GACN,gBAAgB;EAClB;EAEF,IAAI,CAAC,eACH,OAAO;EAET,OAAO;GAAC,WAAW;GAAK,YAAY;EAAa;CACnD;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAKM;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;AACF"}

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -1,4 +1,5 @@
 import type { LogContext } from '@rocicorp/logger';
+import { type JSONObject } from '../../../../shared/src/bigint-json.ts';
 import { TransactionPool } from '../../db/transaction-pool.ts';
 import { type PostgresDB } from '../../types/pg.ts';
 import { type ShardID } from '../../types/shards.ts';
@@ -10,6 +11,11 @@ import type { ReplicatorMode } from '../replicator/replicator.ts';
 import type { Service } from '../service.ts';
 import { type ChangeLogEntry } from './change-log-codec.ts';
 import type { Subscriber } from './subscriber.ts';
+/**
+ * Factory for creating dynamic connection pools used by the Storer to
+ * isolate concurrent transactions into separate pg clients.
+ */
+export type PostgresDBProvider = (applicationName: string, maxConns: number) => PostgresDB;
 export type TuningOptions = {
     backPressureLimitHeapProportion: number;
     statementTimeoutMs: number;
@@ -49,7 +55,7 @@ export type TuningOptions = {
 export declare class Storer implements Service {
     #private;
     readonly id = "storer";
-    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, db: PostgresDB, replicaVersion: string, onCommitted: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs, }: TuningOptions);
+    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, dbProvider: PostgresDBProvider, replicaVersion: string, onCommitted: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs, }: TuningOptions);
     assumeOwnership(purgeLock?: PurgeLock | null): Promise<void>;
     /**
      * The whole of what a stream connection starts from: where to resume, what to
@@ -133,4 +139,37 @@ export declare class PurgeLocker {
     constructor(lc: LogContext, shard: ShardID, db: PostgresDB);
     acquire(): Promise<PurgeLock | null>;
 }
+type TaskKey = JSONObject;
+type TaskDoneFn = () => void;
+/**
+ * Periodic monitor for detecting the pathological condition in which a db
+ * query hangs forever because of a connection problem, such as a half-closed
+ * socket.
+ *
+ * Use this for db calls that recur or accumulate progress within a single
+ * logical operation on the main storer loop (e.g. the queue-processing loop,
+ * or a multi-batch catchup read), where "no progress for N ms" is the
+ * meaningful signal. For a one-off db call -- including ones made before
+ * {@link Storer.run} starts this monitor's polling -- use a plain timeout
+ * (see `Storer.#withTimeout`) instead.
+ *
+ * The failure handler should stop the server. The ProgressMonitor will
+ * assume this and stop itself after calling it.
+ *
+ * Internal to the Storer, but exported for testing.
+ */
+export declare class ProgressMonitor {
+    #private;
+    constructor(lc: LogContext, failureThresholdMs: number, onFailure: (err: Error) => void);
+    start(): void;
+    checkTaskProgress(now: Date): void;
+    /**
+     * Starts a task to be tracked.
+     *
+     * @return a callback to invoke when the task is done.
+     */
+    trackTask(task: TaskKey, now?: Date): TaskDoneFn;
+    stop(): void;
+}
+export {};
 //# sourceMappingURL=storer.d.ts.map

package/out/zero-cache/src/services/change-streamer/storer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,EAIL,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAQ/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAsChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAoBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa;IAiCZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA2BlD;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC;IAoDI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAOzD;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC;QAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IAUF;;;OAGG;IACG,oBAAoB,CACxB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC;IAWpB;;;;;;OAMG;IACH,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,SAAO,GACf,aAAa,CAAC,cAAc,EAAE,CAAC;IAclC,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAOpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IA6FzC;;;OAGG;IACG,GAAG;IAmdT;;;OAGG;IACG,YAAY;IAQlB;;;;OAIG;IACG,IAAI;CAcX;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd"}
+{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,uCAAuC,CAAC;AAM/C,OAAO,EAAiB,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,EAIL,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAQ/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM,KACb,UAAU,CAAC;AAsChB,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBA6BrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,kBAAkB,EAC9B,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa;IAkEZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA8BlD;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC;IAwDI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IASzD;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC;QAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IAWF;;;OAGG;IACG,oBAAoB,CACxB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC;IAWpB;;;;;;OAMG;IACH,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,SAAO,GACf,aAAa,CAAC,cAAc,EAAE,CAAC;IAkBlC,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAMpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IA2EzC;;;OAGG;IACG,GAAG;IAghBT;;;OAGG;IACG,YAAY;IAQlB;;;;OAIG;IACG,IAAI;CAeX;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd;AAKD,KAAK,OAAO,GAAG,UAAU,CAAC;AAE1B,KAAK,UAAU,GAAG,MAAM,IAAI,CAAC;AAE7B;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,eAAe;;gBASxB,EAAE,EAAE,UAAU,EACd,kBAAkB,EAAE,MAAM,EAC1B,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI;IAOjC,KAAK;IASL,iBAAiB,CAAC,GAAG,EAAE,IAAI;IAkB3B;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,OAAa,GAAG,UAAU;IAOtD,IAAI;CAIL"}