@rocicorp/zero 1.8.0-canary.2 → 1.8.0-canary.4

package/out/zero-cache/src/services/change-streamer/vfs-backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"vfs-backup-monitor.js","names":["#lc","#backupURL","#changeStreamer","#source","#probeIntervalMs","#state","#reservations","#watermarks","#purgesBlocked","#cleanupDelayMs","#checkWatermarkTimer","#initBackupLagMetric","#checkWatermark","#scheduleCleanup","#latestBackupWatermark","#lastWatermark","#maxWatermarkUpTo"],"sources":["../../../../../../zero-cache/src/services/change-streamer/vfs-backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n} from '../../observability/metrics.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {VfsBackupWatermark} from '../litestream/vfs-watermark-reader.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\nexport interface VfsBackupWatermarkSource {\n  readWatermark(): Promise<VfsBackupWatermark>;\n  close?(): void;\n}\n\n/**\n * Monitors a Litestream v0.5.x backup by reading Zero's replication state from\n * the backup itself through the Litestream SQLite VFS.\n */\nexport class VfsBackupMonitor implements BackupMonitor {\n  readonly id = 'vfs-backup-monitor';\n  readonly #lc: LogContext;\n  readonly #backupURL: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #source: VfsBackupWatermarkSource;\n  readonly #probeIntervalMs: number;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, VfsBackupWatermark>();\n\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup ' +\n      'watermark could not be read through the Litestream VFS.',\n  });\n\n  #lastWatermark: string = '';\n  #latestBackupWatermark: VfsBackupWatermark | undefined;\n  #cleanupDelayMs: number;\n  #checkWatermarkTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    backupURL: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n    probeIntervalMs: number,\n    source: VfsBackupWatermarkSource,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#backupURL = backupURL;\n    this.#changeStreamer = changeStreamer;\n    this.#source = source;\n    this.#probeIntervalMs = probeIntervalMs;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring v5 backups at ${this.#backupURL} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkWatermarkTimer = setInterval(\n      this.checkWatermarkAndScheduleCleanup,\n      this.#probeIntervalMs,\n    );\n    this.#initBackupLagMetric();\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true): void {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel();\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarkAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermark();\n    } catch (e) {\n      this.#purgesBlocked.add(1, {reason: 'vfs-probe-failed'});\n      this.#lc.warn?.(\n        `unable to read backup watermark through Litestream VFS. ` +\n          `skipping change-log cleanup`,\n        e,\n      );\n      return;\n    }\n\n    this.#scheduleCleanup();\n  };\n\n  async #checkWatermark(): Promise<void> {\n    const watermark = await this.#source.readWatermark();\n    this.#latestBackupWatermark = watermark;\n    if (\n      watermark.watermark > this.#lastWatermark &&\n      !this.#watermarks.has(watermark.watermark)\n    ) {\n      this.#lc.info?.(\n        `observed backup watermark=${watermark.watermark} through ` +\n          `Litestream VFS at ${new Date(watermark.observedAtMs).toISOString()}`,\n        {\n          writeTimeMs: watermark.writeTimeMs,\n          txid: watermark.txid,\n          lagSeconds: watermark.lagSeconds,\n        },\n      );\n      this.#watermarks.set(watermark.watermark, watermark);\n    }\n  }\n\n  #scheduleCleanup(): void {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n\n    const latestConfirmedWatermark = this.#latestBackupWatermark?.watermark;\n    if (latestConfirmedWatermark === undefined) {\n      return;\n    }\n\n    const maxWatermark = this.#maxWatermarkUpTo(\n      Date.now() - this.#cleanupDelayMs,\n      latestConfirmedWatermark,\n    );\n    if (maxWatermark.length === 0) {\n      return;\n    }\n\n    this.#changeStreamer.scheduleCleanup(maxWatermark);\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= maxWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#lastWatermark = maxWatermark;\n  }\n\n  #maxWatermarkUpTo(cutoff: number, latestConfirmedWatermark: string): string {\n    let max = '';\n    for (const [watermark, backupWatermark] of this.#watermarks) {\n      if (\n        watermark <= latestConfirmedWatermark &&\n        backupWatermark.observedAtMs <= cutoff &&\n        watermark > max\n      ) {\n        max = watermark;\n      }\n    }\n    return max;\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkWatermarkTimer);\n    for (const {sub} of this.#reservations.values()) {\n      sub.cancel();\n    }\n    this.#source.close?.();\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric(): void {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackupWatermark = this.#latestBackupWatermark;\n      if (latestBackupWatermark?.writeTimeMs === undefined) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      if (latestBackupWatermark.writeTimeMs === null) {\n        return;\n      }\n      o.observe(\n        Math.max(\n          0,\n          latestBackupWatermark.observedAtMs -\n            latestBackupWatermark.writeTimeMs,\n        ),\n      );\n    });\n  }\n}\n"],"mappings":";;;;;AAaA,IAAM,uBAAuB;;;;;AAgB7B,IAAa,mBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,gCAAyB,IAAI,IAAyB;CACtD,8BAAuB,IAAI,IAAgC;CAE3D,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,wHAEJ,CAAC;CAED,iBAAyB;CACzB;CACA;CACA;CAEA,YACE,IACA,WACA,gBACA,uBACA,iBACA,QACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,aAAa;EAClB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,mBAAmB;EACxB,KAAKK,kBAAkB,KAAK,IAC1B,uBACA,oBACF;CACF;CAEA,MAAqB;EACnB,KAAKT,IAAI,OACP,4BAA4B,KAAKC,WAAW,QACvC,KAAKQ,gBAAgB,kBAC5B;EACA,KAAKC,uBAAuB,YAC1B,KAAK,kCACL,KAAKN,gBACP;EACA,KAAKO,qBAAqB;EAC1B,OAAO,KAAKN,OAAO,QAAQ;CAC7B;CAEA,yBAAyB,QAA+C;EACtE,KAAKL,IAAI,OAAO,oCAAoC,OAAO,WAAW;EACtE,KAAKM,cAAc,IAAI,MAAM,GAAG,IAAI,OAAO;EAE3C,MAAM,MAAM,aAAa,OAAwB,EAC/C,eAAe,KAAK,eAAe,QAAQ,KAAK,EAClD,CAAC;EACD,KAAKA,cAAc,IAAI,QAAQ;GAAC,uBAAO,IAAI,KAAK;GAAG;EAAG,CAAC;EAEvD,KAAUJ,gBACP,kBAAkB,EAClB,MAAK,mBAAkB;GACtB,IAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,KAAKD;IAAY,GAAG;GAAc,CAC/D,CAAC;EACH,CAAC,EACA,OAAM,MAAK;GACV,KAAKD,IAAI,OAAO,gCAAgC,CAAC;GACjD,IAAI,KAAK,CAAC;EACZ,CAAC;EACH,OAAO;CACT;CAEA,eAAe,QAAgB,qBAAqB,MAAY;EAC9D,MAAM,MAAM,KAAKM,cAAc,IAAI,MAAM;EACzC,IAAI,QAAQ,KAAA,GACV;EAEF,KAAKA,cAAc,OAAO,MAAM;EAChC,MAAM,EAAC,OAAO,QAAO;EACrB,IAAI,OAAO;EAEX,IAAI,oBAAoB;GACtB,MAAM,WAAW,KAAK,IAAI,IAAI,MAAM,QAAQ;GAC5C,KAAKN,IAAI,OAAO,2BAA2B,OAAO,MAAM,SAAS,IAAI;GACrE,IAAI,WAAW,KAAKS,iBAAiB;IACnC,KAAKA,kBAAkB;IACvB,KAAKT,IAAI,OAAO,8BAA8B,SAAS,IAAI;GAC7D;EACF;CACF;CAGA,mCAA4C,YAAY;EACtD,IAAI;GACF,MAAM,KAAKY,gBAAgB;EAC7B,SAAS,GAAG;GACV,KAAKJ,eAAe,IAAI,GAAG,EAAC,QAAQ,mBAAkB,CAAC;GACvD,KAAKR,IAAI,OACP,uFAEA,CACF;GACA;EACF;EAEA,KAAKa,iBAAiB;CACxB;CAEA,MAAMD,kBAAiC;EACrC,MAAM,YAAY,MAAM,KAAKT,QAAQ,cAAc;EACnD,KAAKW,yBAAyB;EAC9B,IACE,UAAU,YAAY,KAAKC,kBAC3B,CAAC,KAAKR,YAAY,IAAI,UAAU,SAAS,GACzC;GACA,KAAKP,IAAI,OACP,6BAA6B,UAAU,UAAU,6BAC1B,IAAI,KAAK,UAAU,YAAY,EAAE,YAAY,KACpE;IACE,aAAa,UAAU;IACvB,MAAM,UAAU;IAChB,YAAY,UAAU;GACxB,CACF;GACA,KAAKO,YAAY,IAAI,UAAU,WAAW,SAAS;EACrD;CACF;CAEA,mBAAyB;EACvB,IAAI,KAAKD,cAAc,OAAO,GAAG;GAC/B,KAAKN,IAAI,OACP,6CAA6C,CAAC,GAAG,KAAKM,cAAc,KAAK,CAAC,GAC5E;GACA;EACF;EAEA,MAAM,2BAA2B,KAAKQ,wBAAwB;EAC9D,IAAI,6BAA6B,KAAA,GAC/B;EAGF,MAAM,eAAe,KAAKE,kBACxB,KAAK,IAAI,IAAI,KAAKP,iBAClB,wBACF;EACA,IAAI,aAAa,WAAW,GAC1B;EAGF,KAAKP,gBAAgB,gBAAgB,YAAY;EACjD,KAAK,MAAM,aAAa,KAAKK,YAAY,KAAK,GAC5C,IAAI,aAAa,cACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKQ,iBAAiB;CACxB;CAEA,kBAAkB,QAAgB,0BAA0C;EAC1E,IAAI,MAAM;EACV,KAAK,MAAM,CAAC,WAAW,oBAAoB,KAAKR,aAC9C,IACE,aAAa,4BACb,gBAAgB,gBAAgB,UAChC,YAAY,KAEZ,MAAM;EAGV,OAAO;CACT;CAEA,OAAsB;EACpB,cAAc,KAAKG,oBAAoB;EACvC,KAAK,MAAM,EAAC,SAAQ,KAAKJ,cAAc,OAAO,GAC5C,IAAI,OAAO;EAEb,KAAKH,QAAQ,QAAQ;EACrB,KAAKE,OAAO,KAAK,KAAKL,GAAG;EACzB,OAAO;CACT;CAEA,uBAA6B;EAC3B,iBAAiB,WAAW,cAAc;GACxC,aACE;GAEF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,wBAAwB,KAAKc;GACnC,IAAI,uBAAuB,gBAAgB,KAAA,GAAW;IACpD,KAAKd,IAAI,OACP,8DACF;IACA;GACF;GACA,IAAI,sBAAsB,gBAAgB,MACxC;GAEF,EAAE,QACA,KAAK,IACH,GACA,sBAAsB,eACpB,sBAAsB,WAC1B,CACF;EACF,CAAC;CACH;AACF"}
+{"version":3,"file":"vfs-backup-monitor.js","names":["#lc","#backupURL","#changeStreamer","#source","#probeIntervalMs","#state","#reservations","#watermarks","#purgesBlocked","#cleanupDelayMs","#checkWatermarkTimer","#initBackupLagMetric","#checkWatermark","#scheduleCleanup","#readWatermarkTimed","#latestBackupWatermark","#lastWatermark","#maxWatermarkUpTo"],"sources":["../../../../../../zero-cache/src/services/change-streamer/vfs-backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n} from '../../observability/metrics.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamBackupVerificationDuration,\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {VfsBackupWatermark} from '../litestream/vfs-watermark-reader.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\nexport interface VfsBackupWatermarkSource {\n  readWatermark(): Promise<VfsBackupWatermark>;\n  close?(): void;\n}\n\n/**\n * Monitors a Litestream v0.5.x backup by reading Zero's replication state from\n * the backup itself through the Litestream SQLite VFS.\n */\nexport class VfsBackupMonitor implements BackupMonitor {\n  readonly id = 'vfs-backup-monitor';\n  readonly #lc: LogContext;\n  readonly #backupURL: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #source: VfsBackupWatermarkSource;\n  readonly #probeIntervalMs: number;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, VfsBackupWatermark>();\n\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup ' +\n      'watermark could not be read through the Litestream VFS.',\n  });\n\n  #lastWatermark: string = '';\n  #latestBackupWatermark: VfsBackupWatermark | undefined;\n  #cleanupDelayMs: number;\n  #checkWatermarkTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    backupURL: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n    probeIntervalMs: number,\n    source: VfsBackupWatermarkSource,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#backupURL = backupURL;\n    this.#changeStreamer = changeStreamer;\n    this.#source = source;\n    this.#probeIntervalMs = probeIntervalMs;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring v5 backups at ${this.#backupURL} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkWatermarkTimer = setInterval(\n      this.checkWatermarkAndScheduleCleanup,\n      this.#probeIntervalMs,\n    );\n    this.#initBackupLagMetric();\n    return this.#state.stopped();\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true): void {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel();\n    const duration = Date.now() - start.getTime();\n    litestreamSnapshotReservationDuration().recordMs(duration, {\n      ...litestreamMonitorMetricAttrs(this.#backupURL, 'v5', 'view_syncer'),\n      result: updateCleanupDelay ? 'success' : 'cancelled',\n    });\n\n    if (updateCleanupDelay) {\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarkAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermark();\n    } catch (e) {\n      this.#purgesBlocked.add(1, {reason: 'vfs-probe-failed'});\n      this.#lc.warn?.(\n        `unable to read backup watermark through Litestream VFS. ` +\n          `skipping change-log cleanup`,\n        e,\n      );\n      return;\n    }\n\n    this.#scheduleCleanup();\n  };\n\n  async #checkWatermark(): Promise<void> {\n    const watermark = await this.#readWatermarkTimed();\n    this.#latestBackupWatermark = watermark;\n    if (\n      watermark.watermark > this.#lastWatermark &&\n      !this.#watermarks.has(watermark.watermark)\n    ) {\n      this.#lc.info?.(\n        `observed backup watermark=${watermark.watermark} through ` +\n          `Litestream VFS at ${new Date(watermark.observedAtMs).toISOString()}`,\n        {\n          writeTimeMs: watermark.writeTimeMs,\n          txid: watermark.txid,\n          lagSeconds: watermark.lagSeconds,\n        },\n      );\n      this.#watermarks.set(watermark.watermark, watermark);\n    }\n  }\n\n  async #readWatermarkTimed(): Promise<VfsBackupWatermark> {\n    const start = performance.now();\n    let result: 'success' | 'error' = 'error';\n    try {\n      const ret = await this.#source.readWatermark();\n      result = 'success';\n      return ret;\n    } finally {\n      litestreamBackupVerificationDuration().recordMs(\n        performance.now() - start,\n        {\n          ...litestreamMonitorMetricAttrs(\n            this.#backupURL,\n            'v5',\n            'replication_manager',\n          ),\n          result,\n        },\n      );\n    }\n  }\n\n  #scheduleCleanup(): void {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n\n    const latestConfirmedWatermark = this.#latestBackupWatermark?.watermark;\n    if (latestConfirmedWatermark === undefined) {\n      return;\n    }\n\n    const maxWatermark = this.#maxWatermarkUpTo(\n      Date.now() - this.#cleanupDelayMs,\n      latestConfirmedWatermark,\n    );\n    if (maxWatermark.length === 0) {\n      return;\n    }\n\n    this.#changeStreamer.scheduleCleanup(maxWatermark);\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= maxWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#lastWatermark = maxWatermark;\n  }\n\n  #maxWatermarkUpTo(cutoff: number, latestConfirmedWatermark: string): string {\n    let max = '';\n    for (const [watermark, backupWatermark] of this.#watermarks) {\n      if (\n        watermark <= latestConfirmedWatermark &&\n        backupWatermark.observedAtMs <= cutoff &&\n        watermark > max\n      ) {\n        max = watermark;\n      }\n    }\n    return max;\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkWatermarkTimer);\n    for (const {sub} of this.#reservations.values()) {\n      sub.cancel();\n    }\n    this.#source.close?.();\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric(): void {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackupWatermark = this.#latestBackupWatermark;\n      if (latestBackupWatermark?.writeTimeMs === undefined) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      if (latestBackupWatermark.writeTimeMs === null) {\n        return;\n      }\n      o.observe(\n        Math.max(\n          0,\n          latestBackupWatermark.observedAtMs -\n            latestBackupWatermark.writeTimeMs,\n        ),\n      );\n    });\n  }\n}\n"],"mappings":";;;;;;AAkBA,IAAM,uBAAuB;;;;;AAgB7B,IAAa,mBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,gCAAyB,IAAI,IAAyB;CACtD,8BAAuB,IAAI,IAAgC;CAE3D,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,wHAEJ,CAAC;CAED,iBAAyB;CACzB;CACA;CACA;CAEA,YACE,IACA,WACA,gBACA,uBACA,iBACA,QACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,aAAa;EAClB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,mBAAmB;EACxB,KAAKK,kBAAkB,KAAK,IAC1B,uBACA,oBACF;CACF;CAEA,MAAqB;EACnB,KAAKT,IAAI,OACP,4BAA4B,KAAKC,WAAW,QACvC,KAAKQ,gBAAgB,kBAC5B;EACA,KAAKC,uBAAuB,YAC1B,KAAK,kCACL,KAAKN,gBACP;EACA,KAAKO,qBAAqB;EAC1B,OAAO,KAAKN,OAAO,QAAQ;CAC7B;CAEA,yBAAyB,QAA+C;EACtE,KAAKL,IAAI,OAAO,oCAAoC,OAAO,WAAW;EACtE,KAAKM,cAAc,IAAI,MAAM,GAAG,IAAI,OAAO;EAE3C,MAAM,MAAM,aAAa,OAAwB,EAC/C,eAAe,KAAK,eAAe,QAAQ,KAAK,EAClD,CAAC;EACD,KAAKA,cAAc,IAAI,QAAQ;GAAC,uBAAO,IAAI,KAAK;GAAG;EAAG,CAAC;EAEvD,KAAUJ,gBACP,kBAAkB,EAClB,MAAK,mBAAkB;GACtB,IAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,KAAKD;IAAY,GAAG;GAAc,CAC/D,CAAC;EACH,CAAC,EACA,OAAM,MAAK;GACV,KAAKD,IAAI,OAAO,gCAAgC,CAAC;GACjD,IAAI,KAAK,CAAC;EACZ,CAAC;EACH,OAAO;CACT;CAEA,eAAe,QAAgB,qBAAqB,MAAY;EAC9D,MAAM,MAAM,KAAKM,cAAc,IAAI,MAAM;EACzC,IAAI,QAAQ,KAAA,GACV;EAEF,KAAKA,cAAc,OAAO,MAAM;EAChC,MAAM,EAAC,OAAO,QAAO;EACrB,IAAI,OAAO;EACX,MAAM,WAAW,KAAK,IAAI,IAAI,MAAM,QAAQ;EAC5C,sCAAsC,EAAE,SAAS,UAAU;GACzD,GAAG,6BAA6B,KAAKL,YAAY,MAAM,aAAa;GACpE,QAAQ,qBAAqB,YAAY;EAC3C,CAAC;EAED,IAAI,oBAAoB;GACtB,KAAKD,IAAI,OAAO,2BAA2B,OAAO,MAAM,SAAS,IAAI;GACrE,IAAI,WAAW,KAAKS,iBAAiB;IACnC,KAAKA,kBAAkB;IACvB,KAAKT,IAAI,OAAO,8BAA8B,SAAS,IAAI;GAC7D;EACF;CACF;CAGA,mCAA4C,YAAY;EACtD,IAAI;GACF,MAAM,KAAKY,gBAAgB;EAC7B,SAAS,GAAG;GACV,KAAKJ,eAAe,IAAI,GAAG,EAAC,QAAQ,mBAAkB,CAAC;GACvD,KAAKR,IAAI,OACP,uFAEA,CACF;GACA;EACF;EAEA,KAAKa,iBAAiB;CACxB;CAEA,MAAMD,kBAAiC;EACrC,MAAM,YAAY,MAAM,KAAKE,oBAAoB;EACjD,KAAKC,yBAAyB;EAC9B,IACE,UAAU,YAAY,KAAKC,kBAC3B,CAAC,KAAKT,YAAY,IAAI,UAAU,SAAS,GACzC;GACA,KAAKP,IAAI,OACP,6BAA6B,UAAU,UAAU,6BAC1B,IAAI,KAAK,UAAU,YAAY,EAAE,YAAY,KACpE;IACE,aAAa,UAAU;IACvB,MAAM,UAAU;IAChB,YAAY,UAAU;GACxB,CACF;GACA,KAAKO,YAAY,IAAI,UAAU,WAAW,SAAS;EACrD;CACF;CAEA,MAAMO,sBAAmD;EACvD,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAKX,QAAQ,cAAc;GAC7C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,EAAE,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAKF,YACL,MACA,qBACF;IACA;GACF,CACF;EACF;CACF;CAEA,mBAAyB;EACvB,IAAI,KAAKK,cAAc,OAAO,GAAG;GAC/B,KAAKN,IAAI,OACP,6CAA6C,CAAC,GAAG,KAAKM,cAAc,KAAK,CAAC,GAC5E;GACA;EACF;EAEA,MAAM,2BAA2B,KAAKS,wBAAwB;EAC9D,IAAI,6BAA6B,KAAA,GAC/B;EAGF,MAAM,eAAe,KAAKE,kBACxB,KAAK,IAAI,IAAI,KAAKR,iBAClB,wBACF;EACA,IAAI,aAAa,WAAW,GAC1B;EAGF,KAAKP,gBAAgB,gBAAgB,YAAY;EACjD,KAAK,MAAM,aAAa,KAAKK,YAAY,KAAK,GAC5C,IAAI,aAAa,cACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKS,iBAAiB;CACxB;CAEA,kBAAkB,QAAgB,0BAA0C;EAC1E,IAAI,MAAM;EACV,KAAK,MAAM,CAAC,WAAW,oBAAoB,KAAKT,aAC9C,IACE,aAAa,4BACb,gBAAgB,gBAAgB,UAChC,YAAY,KAEZ,MAAM;EAGV,OAAO;CACT;CAEA,OAAsB;EACpB,cAAc,KAAKG,oBAAoB;EACvC,KAAK,MAAM,EAAC,SAAQ,KAAKJ,cAAc,OAAO,GAC5C,IAAI,OAAO;EAEb,KAAKH,QAAQ,QAAQ;EACrB,KAAKE,OAAO,KAAK,KAAKL,GAAG;EACzB,OAAO;CACT;CAEA,uBAA6B;EAC3B,iBAAiB,WAAW,cAAc;GACxC,aACE;GAEF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,wBAAwB,KAAKe;GACnC,IAAI,uBAAuB,gBAAgB,KAAA,GAAW;IACpD,KAAKf,IAAI,OACP,8DACF;IACA;GACF;GACA,IAAI,sBAAsB,gBAAgB,MACxC;GAEF,EAAE,QACA,KAAK,IACH,GACA,sBAAsB,eACpB,sBAAsB,WAC1B,CACF;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/litestream/commands.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAM3D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAe5D,KAAK,kBAAkB,GAAG;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;gBAErC,SAAS,EAAE,MAAM,GAAG,SAAS;CAG1C;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,kBAAkB,EAAE,kBAAkB,GAAG,IAAI,iBA4C9C;AAoKD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAQd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAgBf;AAqDD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}
+{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAM3D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AA+B5D,KAAK,kBAAkB,GAAG;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAUF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;gBAErC,SAAS,EAAE,MAAM,GAAG,SAAS;CAG1C;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,kBAAkB,EAAE,kBAAkB,GAAG,IAAI,iBAqE9C;AA0ND,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA6DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}

package/out/zero-cache/src/services/litestream/commands.js

@@ -8,9 +8,10 @@ import { StatementRunner } from "../../db/statements.js";
 import { deleteLiteDB } from "../../db/delete-lite-db.js";
 import { assertDatabaseIntegrity } from "../../db/migration-lite.js";
 import { ChangeStreamerHttpClient } from "../change-streamer/change-streamer-http.js";
+import { litestreamBackupListDuration, litestreamBackupMetricAttrs, litestreamBackupProcessDuration, litestreamBackupProcessMetricAttrs, litestreamBackupProcessRuns, litestreamRestoreAttempts, litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreProcessDuration, litestreamRestoreRuns, litestreamRestoreValidationDuration, litestreamRestoreWaitDuration, litestreamRestoredDbBytes } from "./metrics.js";
 import { resolver } from "@rocicorp/resolver";
 import { spawn } from "node:child_process";
-import { existsSync } from "node:fs";
+import { existsSync, statSync } from "node:fs";
 //#region ../zero-cache/src/services/litestream/commands.ts
 var RETRY_INTERVAL_MS = 3e3;
 var BackupNotFoundException = class extends Error {
@@ -26,21 +27,46 @@ var BackupNotFoundException = class extends Error {
 *        retrieved from the replication-manager via the snapshot protocol.
 */
 async function restoreReplica(lc, config, replicaConstraints) {
+	const start = performance.now();
+	const role = replicaConstraints ? "replication_manager" : "view_syncer";
+	let backupURL = config.litestream.backupURL;
+	let result;
 	let consecutiveErrors = 0;
-	for (;;) {
-		try {
-			if (await tryRestore(lc, config, replicaConstraints)) return;
-			consecutiveErrors = 0;
-		} catch (e) {
-			if (++consecutiveErrors <= 1) {
-				lc.warn?.(`restore attempt failed. retrying once`, e);
-				continue;
+	try {
+		for (;;) {
+			try {
+				const attempt = await tryRestore(lc, config, replicaConstraints, role);
+				backupURL = attempt.backupURL;
+				if (attempt.restored) {
+					result = attempt.result;
+					return;
+				}
+				consecutiveErrors = 0;
+				if (replicaConstraints) {
+					result = attempt.result;
+					throw new BackupNotFoundException(config.litestream.backupURL);
+				}
+			} catch (e) {
+				if (e instanceof BackupNotFoundException) throw e;
+				if (++consecutiveErrors <= 1) {
+					lc.warn?.(`restore attempt failed. retrying once`, e);
+					continue;
+				}
+				throw e;
 			}
-			throw e;
+			lc.info?.(`replica not found. retrying in ${RETRY_INTERVAL_MS / 1e3} seconds`);
+			await sleep(RETRY_INTERVAL_MS);
 		}
-		if (replicaConstraints) throw new BackupNotFoundException(config.litestream.backupURL);
-		lc.info?.(`replica not found. retrying in ${RETRY_INTERVAL_MS / 1e3} seconds`);
-		await sleep(RETRY_INTERVAL_MS);
+	} catch (e) {
+		if (e instanceof BackupNotFoundException && result === void 0) result = "no_backup";
+		throw e;
+	} finally {
+		const labels = {
+			...litestreamRestoreMetricAttrs(config, role, backupURL),
+			result: result ?? "error"
+		};
+		litestreamRestoreRuns().add(1, labels);
+		litestreamRestoreDuration().recordMs(performance.now() - start, labels);
 	}
 }
 function getLitestream(mode, config, logLevelOverride, backupURLOverride) {
@@ -67,42 +93,100 @@ function getLitestream(mode, config, logLevelOverride, backupURLOverride) {
 		}
 	};
 }
-async function tryRestore(lc, config, replicaConstraints) {
+async function tryRestore(lc, config, replicaConstraints, role) {
 	let snapshotStatus;
 	if (!replicaConstraints) {
+		const waitStart = performance.now();
 		snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);
+		litestreamRestoreWaitDuration().recordMs(performance.now() - waitStart, litestreamRestoreMetricAttrs(config, role, snapshotStatus.backupURL));
 		lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);
 		replicaConstraints = snapshotStatus;
 	}
-	const { litestream, env } = getLitestream("restore", config, "debug", snapshotStatus?.backupURL);
-	const { restoreParallelism: parallelism } = config.litestream;
-	const proc = spawn(litestream, [
-		"restore",
-		"-if-db-not-exists",
-		"-if-replica-exists",
-		"-parallelism",
-		String(parallelism),
-		config.replica.file
-	], {
-		env,
-		stdio: "inherit",
-		windowsHide: true
-	});
-	const { promise, resolve, reject } = resolver();
-	proc.on("error", reject);
-	proc.on("close", (code, signal) => {
-		if (signal) reject(`litestream killed with ${signal}`);
-		else if (code !== 0) reject(`litestream exited with code ${code}`);
-		else resolve();
-	});
-	await promise;
-	if (!existsSync(config.replica.file)) return false;
-	if (!replicaIsValid(lc, config.replica.file, replicaConstraints)) {
-		lc.info?.(`Deleting local replica and retrying restore`);
-		deleteLiteDB(config.replica.file);
-		return false;
+	const backupURL = snapshotStatus?.backupURL ?? config.litestream.backupURL;
+	const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);
+	let result = "error";
+	try {
+		const replicaExistedBeforeRestore = existsSync(config.replica.file);
+		const { litestream, env } = getLitestream("restore", config, "debug", snapshotStatus?.backupURL);
+		const { restoreParallelism: parallelism, multipartConcurrency, multipartSize } = config.litestream;
+		lc.info?.(`starting litestream restore`, {
+			restoreParallelism: parallelism,
+			multipartConcurrency,
+			multipartSize
+		});
+		const proc = spawn(litestream, [
+			"restore",
+			"-if-db-not-exists",
+			"-if-replica-exists",
+			"-parallelism",
+			String(parallelism),
+			config.replica.file
+		], {
+			env,
+			stdio: "inherit",
+			windowsHide: true
+		});
+		const { promise, resolve, reject } = resolver();
+		proc.on("error", reject);
+		proc.on("close", (code, signal) => {
+			if (signal) reject(`litestream killed with ${signal}`);
+			else if (code !== 0) reject(`litestream exited with code ${code}`);
+			else resolve();
+		});
+		const processStart = performance.now();
+		try {
+			await promise;
+			litestreamRestoreProcessDuration().recordMs(performance.now() - processStart, {
+				...attrs,
+				result: "success"
+			});
+		} catch (e) {
+			litestreamRestoreProcessDuration().recordMs(performance.now() - processStart, {
+				...attrs,
+				result: "error"
+			});
+			throw e;
+		}
+		if (!existsSync(config.replica.file)) {
+			result = "no_backup";
+			return {
+				restored: false,
+				backupURL,
+				result
+			};
+		}
+		const validationStart = performance.now();
+		const valid = replicaIsValid(lc, config.replica.file, replicaConstraints);
+		litestreamRestoreValidationDuration().recordMs(performance.now() - validationStart, {
+			...attrs,
+			result: valid ? "success" : "invalid_replica"
+		});
+		if (!valid) {
+			result = "invalid_replica";
+			lc.info?.(`Deleting local replica and retrying restore`);
+			deleteLiteDB(config.replica.file);
+			return {
+				restored: false,
+				backupURL,
+				result
+			};
+		}
+		result = "success";
+		if (!replicaExistedBeforeRestore) litestreamRestoredDbBytes().add(statSync(config.replica.file).size, {
+			...attrs,
+			result: "success"
+		});
+		return {
+			restored: true,
+			backupURL,
+			result
+		};
+	} finally {
+		litestreamRestoreAttempts().add(1, {
+			...attrs,
+			result
+		});
 	}
-	return true;
 }
 function replicaIsValid(lc, replica, constraints) {
 	let db;
@@ -129,12 +213,40 @@ function replicaIsValid(lc, replica, constraints) {
 }
 function startReplicaBackupProcess(lc, config) {
 	const { litestream, env } = getLitestream("replicate", config);
+	const attrs = litestreamBackupProcessMetricAttrs(config);
 	lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);
-	return spawn(litestream, ["replicate"], {
+	const start = performance.now();
+	const proc = spawn(litestream, ["replicate"], {
 		env,
 		stdio: "inherit",
 		windowsHide: true
 	});
+	let recorded = false;
+	const record = (result) => {
+		if (recorded) return;
+		recorded = true;
+		const labels = {
+			...attrs,
+			result
+		};
+		litestreamBackupProcessRuns().add(1, labels);
+		litestreamBackupProcessDuration().recordMs(performance.now() - start, labels);
+	};
+	proc.on("error", (e) => {
+		lc.warn?.(`litestream backup process error`, e);
+		record("error");
+	});
+	proc.on("close", (code, signal) => {
+		if (signal) {
+			lc.info?.(`litestream backup process stopped`, { signal });
+			record("stopped");
+		} else if (code === 0) record("success");
+		else {
+			lc.warn?.(`litestream backup process exited with code ${code}`);
+			record("error");
+		}
+	});
+	return proc;
 }
 var LIST_BACKUP_TIMEOUT_MS = 3e4;
 var wsRe = /\s+/;
@@ -170,6 +282,8 @@ async function getLastBackupTime(lc, config) {
 * ```
 */
 async function listBackupCreatedTimes(lc, config, command) {
+	const start = performance.now();
+	let result = "error";
 	const { litestream, env } = getLitestream("replicate", config);
 	const proc = spawn(litestream, [command, config.replica.file], {
 		env,
@@ -191,16 +305,22 @@ async function listBackupCreatedTimes(lc, config, command) {
 		else resolve(stdout);
 	});
 	const timeout = setTimeout(() => {
+		result = "timeout";
 		reject(/* @__PURE__ */ new Error(`timed out listing backup state (litestream ${command})`));
 		proc.kill("SIGKILL");
 	}, LIST_BACKUP_TIMEOUT_MS);
-	let output;
 	try {
-		output = await promise;
+		const times = parseBackupCreatedTimes(lc, command, await promise);
+		result = times.length ? "success" : "empty";
+		return times;
 	} finally {
 		clearTimeout(timeout);
+		litestreamBackupListDuration().recordMs(performance.now() - start, {
+			...litestreamBackupMetricAttrs(config),
+			command,
+			result
+		});
 	}
-	return parseBackupCreatedTimes(lc, command, output);
 }
 /**
 * Parses the `created` column (the last, RFC3339-formatted column) from the

package/out/zero-cache/src/services/litestream/commands.js.map

@@ -1 +1 @@
-{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {assertDatabaseIntegrity} from '../../db/migration-lite.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\nconst RETRY_INTERVAL_MS = 3000;\n\ntype ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * @param replicaConstraints The constraints of the restored backup when\n *        restoring for the change-streamer (replication-manager). For the\n *        view-syncer, this should be unspecified so that the constraints are\n *        retrieved from the replication-manager via the snapshot protocol.\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  // View-syncers (no replicaConstraints) wait indefinitely for the\n  // replication-manager to publish a restorable backup. On a fresh stack the\n  // first backup is not durable until the initial sync completes and litestream\n  // uploads the initial snapshot, which can take many minutes for a large\n  // replica. The platform's startup probe budget (which scales with replica\n  // size) is the backstop, so restoreReplica must not impose its own shorter\n  // cap and self-terminate while the backup is still being produced.\n  //\n  // `consecutiveErrors` distinguishes a transient restore *error* (e.g. the\n  // `replicate` process compacting a snapshot mid-restore) — which is retried\n  // once — from a \"backup not found\" result, which is expected while waiting.\n  let consecutiveErrors = 0;\n  for (;;) {\n    try {\n      if (await tryRestore(lc, config, replicaConstraints)) {\n        return;\n      }\n      consecutiveErrors = 0;\n    } catch (e) {\n      if (++consecutiveErrors <= 1) {\n        // A restore will fail if the `replicate` process creates a new\n        // snapshot (and compacts old files) at the same time. Snapshots are\n        // infrequent (e.g. once every 12 hours), and the scenario is\n        // recoverable with a retry.\n        lc.warn?.(`restore attempt failed. retrying once`, e);\n        continue;\n      }\n      // If it fails again on the retry, though, bail.\n      throw e;\n    }\n    if (replicaConstraints) {\n      // The replication-manager restores against explicit constraints, so a\n      // missing backup is fatal (e.g. the litestream URL was purposefully\n      // changed to force a resync). Only the view-syncer (no constraints)\n      // waits for a backup to appear.\n      throw new BackupNotFoundException(config.litestream.backupURL);\n    }\n    lc.info?.(\n      `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n    );\n    await sleep(RETRY_INTERVAL_MS);\n  }\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupURL,\n    logLevel,\n    configPath,\n    endpoint,\n    region,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  const litestream =\n    // The v0.5.8+ litestream executable can restore from either the new LTX\n    // format or the legacy WAL format, allowing forwards-compatibility /\n    // rollback safety with zero-cache versions that backup to LTX.\n    (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n    },\n  };\n}\n\nasync function tryRestore(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (!replicaConstraints) {\n    // view-syncers fetch replica constraints from the replication-manager\n    // via the snapshot protocol.\n    snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n    replicaConstraints = snapshotStatus;\n  }\n\n  const {litestream, env} = getLitestream(\n    'restore',\n    config,\n    'debug', // Include all output from `litestream restore`, as it's minimal.\n    snapshotStatus?.backupURL,\n  );\n  const {restoreParallelism: parallelism} = config.litestream;\n  const proc = spawn(\n    litestream,\n    [\n      'restore',\n      '-if-db-not-exists',\n      '-if-replica-exists',\n      '-parallelism',\n      String(parallelism),\n      config.replica.file,\n    ],\n    {env, stdio: 'inherit', windowsHide: true},\n  );\n  const {promise, resolve, reject} = resolver();\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(`litestream killed with ${signal}`);\n    } else if (code !== 0) {\n      reject(`litestream exited with code ${code}`);\n    } else {\n      resolve();\n    }\n  });\n  await promise;\n  if (!existsSync(config.replica.file)) {\n    return false;\n  }\n  if (!replicaIsValid(lc, config.replica.file, replicaConstraints)) {\n    lc.info?.(`Deleting local replica and retrying restore`);\n    deleteLiteDB(config.replica.file);\n    return false;\n  }\n  return true;\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints,\n) {\n  let db: Database | undefined;\n  try {\n    db = new Database(lc, replica);\n    assertDatabaseIntegrity(lc, 'restored replica', db);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== constraints.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n        constraints,\n      );\n      return false;\n    }\n    if (watermark < constraints.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n      constraints,\n    );\n    return true;\n  } catch (e) {\n    lc.error?.('Error while validating restored replica', e);\n    return false;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: ZeroConfig,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config);\n  lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n  return spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, 'snapshots'),\n    listBackupCreatedTimes(lc, config, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${config.litestream.backupURL}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: ZeroConfig,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const {litestream, env} = getLitestream('replicate', config);\n  const proc = spawn(litestream, [command, config.replica.file], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  let output: string;\n  try {\n    output = await promise;\n  } finally {\n    clearTimeout(timeout);\n  }\n\n  return parseBackupCreatedTimes(lc, command, output);\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // backup monitor returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;;AAsBA,IAAM,oBAAoB;AAO1B,IAAa,0BAAb,cAA6C,MAAM;CACjD,OAAgB,OAAO;CAEvB,YAAY,WAA+B;EACzC,MAAM,uBAAuB,WAAW;CAC1C;AACF;;;;;;;AAQA,eAAsB,eACpB,IACA,QACA,oBACA;CAYA,IAAI,oBAAoB;CACxB,SAAS;EACP,IAAI;GACF,IAAI,MAAM,WAAW,IAAI,QAAQ,kBAAkB,GACjD;GAEF,oBAAoB;EACtB,SAAS,GAAG;GACV,IAAI,EAAE,qBAAqB,GAAG;IAK5B,GAAG,OAAO,yCAAyC,CAAC;IACpD;GACF;GAEA,MAAM;EACR;EACA,IAAI,oBAKF,MAAM,IAAI,wBAAwB,OAAO,WAAW,SAAS;EAE/D,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;EACA,MAAM,MAAM,iBAAiB;CAC/B;AACF;AAEA,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,QACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;CAQzE,OAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,iCAAiC;EAGlD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,SAAS;IAClE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,6BACF;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAe,WACb,IACA,QACA,oBACA;CACA,IAAI;CACJ,IAAI,CAAC,oBAAoB;EAGvB,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;EAC7D,GAAG,OAAO,yBAAyB,eAAe,WAAW;EAC7D,qBAAqB;CACvB;CAEA,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,SAClB;CACA,MAAM,EAAC,oBAAoB,gBAAe,OAAO;CACjD,MAAM,OAAO,MACX,YACA;EACE;EACA;EACA;EACA;EACA,OAAO,WAAW;EAClB,OAAO,QAAQ;CACjB,GACA;EAAC;EAAK,OAAO;EAAW,aAAa;CAAI,CAC3C;CACA,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;CAC5C,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;OACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;OAE5C,QAAQ;CAEZ,CAAC;CACD,MAAM;CACN,IAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GACjC,OAAO;CAET,IAAI,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,kBAAkB,GAAG;EAChE,GAAG,OAAO,6CAA6C;EACvD,aAAa,OAAO,QAAQ,IAAI;EAChC,OAAO;CACT;CACA,OAAO;AACT;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,wBAAwB,IAAI,oBAAoB,EAAE;EAClD,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,mBAAmB,YAAY,gBAAgB;GACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;GACA,OAAO;EACT;EACA,IAAI,YAAY,YAAY,cAAc;GACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;GACA,OAAO;EACT;EACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,GAAG,OAAO,iCAAiC,OAAO,WAAW,WAAW;CACxE,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EACtC;EACA,OAAO;EACP,aAAa;CACf,CAAC;AACH;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,WAAW,GAC9C,uBAAuB,IAAI,QAAQ,KAAK,CAC1C,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WAAW,WAC9D;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,SACiB;CACjB,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,OAAO,QAAQ,IAAI,GAAG;EAC7D;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;CACJ,IAAI;EACF,SAAS,MAAM;CACjB,UAAU;EACR,aAAa,OAAO;CACtB;CAEA,OAAO,wBAAwB,IAAI,SAAS,MAAM;AACpD;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,4BACP,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,iBAAiB,MAAM;CACvB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}
+{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, statSync} from 'node:fs';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {assertDatabaseIntegrity} from '../../db/migration-lite.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreRuns,\n  litestreamRestoreValidationDuration,\n  litestreamRestoreWaitDuration,\n  type LitestreamRole,\n} from './metrics.ts';\n\nconst RETRY_INTERVAL_MS = 3000;\n\ntype ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\ntype RestoreResult = 'success' | 'no_backup' | 'invalid_replica' | 'error';\n\ntype RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n};\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * @param replicaConstraints The constraints of the restored backup when\n *        restoring for the change-streamer (replication-manager). For the\n *        view-syncer, this should be unspecified so that the constraints are\n *        retrieved from the replication-manager via the snapshot protocol.\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  const start = performance.now();\n  const role: LitestreamRole = replicaConstraints\n    ? 'replication_manager'\n    : 'view_syncer';\n  let backupURL = config.litestream.backupURL;\n  let result: RestoreResult | undefined;\n  // View-syncers (no replicaConstraints) wait indefinitely for the\n  // replication-manager to publish a restorable backup. On a fresh stack the\n  // first backup is not durable until the initial sync completes and litestream\n  // uploads the initial snapshot, which can take many minutes for a large\n  // replica. The platform's startup probe budget (which scales with replica\n  // size) is the backstop, so restoreReplica must not impose its own shorter\n  // cap and self-terminate while the backup is still being produced.\n  //\n  // `consecutiveErrors` distinguishes a transient restore *error* (e.g. the\n  // `replicate` process compacting a snapshot mid-restore) — which is retried\n  // once — from a \"backup not found\" result, which is expected while waiting.\n  let consecutiveErrors = 0;\n  try {\n    for (;;) {\n      try {\n        const attempt = await tryRestore(lc, config, replicaConstraints, role);\n        backupURL = attempt.backupURL;\n        if (attempt.restored) {\n          result = attempt.result;\n          return;\n        }\n        consecutiveErrors = 0;\n        if (replicaConstraints) {\n          // The replication-manager restores against explicit constraints, so a\n          // missing backup is fatal (e.g. the litestream URL was purposefully\n          // changed to force a resync). Only the view-syncer (no constraints)\n          // waits for a backup to appear.\n          result = attempt.result;\n          throw new BackupNotFoundException(config.litestream.backupURL);\n        }\n      } catch (e) {\n        if (e instanceof BackupNotFoundException) {\n          throw e;\n        }\n        if (++consecutiveErrors <= 1) {\n          // A restore will fail if the `replicate` process creates a new\n          // snapshot (and compacts old files) at the same time. Snapshots are\n          // infrequent (e.g. once every 12 hours), and the scenario is\n          // recoverable with a retry.\n          lc.warn?.(`restore attempt failed. retrying once`, e);\n          continue;\n        }\n        // If it fails again on the retry, though, bail.\n        throw e;\n      }\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } catch (e) {\n    if (e instanceof BackupNotFoundException && result === undefined) {\n      result = 'no_backup';\n    }\n    throw e;\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupURL,\n    logLevel,\n    configPath,\n    endpoint,\n    region,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  const litestream =\n    // The v0.5.8+ litestream executable can restore from either the new LTX\n    // format or the legacy WAL format, allowing forwards-compatibility /\n    // rollback safety with zero-cache versions that backup to LTX.\n    (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n    },\n  };\n}\n\nasync function tryRestore(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n  role: LitestreamRole,\n): Promise<RestoreAttempt> {\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (!replicaConstraints) {\n    // view-syncers fetch replica constraints from the replication-manager\n    // via the snapshot protocol.\n    const waitStart = performance.now();\n    snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n    litestreamRestoreWaitDuration().recordMs(\n      performance.now() - waitStart,\n      litestreamRestoreMetricAttrs(config, role, snapshotStatus.backupURL),\n    );\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n    replicaConstraints = snapshotStatus;\n  }\n\n  const backupURL = snapshotStatus?.backupURL ?? config.litestream.backupURL;\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    const replicaExistedBeforeRestore = existsSync(config.replica.file);\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n      snapshotStatus?.backupURL,\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config.litestream;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        config.replica.file,\n      ],\n      {env, stdio: 'inherit', windowsHide: true},\n    );\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      throw e;\n    }\n    if (!existsSync(config.replica.file)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result};\n    }\n    const validationStart = performance.now();\n    const valid = replicaIsValid(lc, config.replica.file, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: valid ? 'success' : 'invalid_replica'},\n    );\n    if (!valid) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(config.replica.file);\n      return {restored: false, backupURL, result};\n    }\n    result = 'success';\n    if (!replicaExistedBeforeRestore) {\n      litestreamRestoredDbBytes().add(statSync(config.replica.file).size, {\n        ...attrs,\n        result: 'success',\n      });\n    }\n    return {restored: true, backupURL, result};\n  } finally {\n    litestreamRestoreAttempts().add(1, {\n      ...attrs,\n      result,\n    });\n  }\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints,\n) {\n  let db: Database | undefined;\n  try {\n    db = new Database(lc, replica);\n    assertDatabaseIntegrity(lc, 'restored replica', db);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== constraints.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n        constraints,\n      );\n      return false;\n    }\n    if (watermark < constraints.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n      constraints,\n    );\n    return true;\n  } catch (e) {\n    lc.error?.('Error while validating restored replica', e);\n    return false;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: ZeroConfig,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, 'snapshots'),\n    listBackupCreatedTimes(lc, config, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${config.litestream.backupURL}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: ZeroConfig,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config);\n  const proc = spawn(litestream, [command, config.replica.file], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // backup monitor returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;;;AAsCA,IAAM,oBAAoB;AAe1B,IAAa,0BAAb,cAA6C,MAAM;CACjD,OAAgB,OAAO;CAEvB,YAAY,WAA+B;EACzC,MAAM,uBAAuB,WAAW;CAC1C;AACF;;;;;;;AAQA,eAAsB,eACpB,IACA,QACA,oBACA;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAuB,qBACzB,wBACA;CACJ,IAAI,YAAY,OAAO,WAAW;CAClC,IAAI;CAYJ,IAAI,oBAAoB;CACxB,IAAI;EACF,SAAS;GACP,IAAI;IACF,MAAM,UAAU,MAAM,WAAW,IAAI,QAAQ,oBAAoB,IAAI;IACrE,YAAY,QAAQ;IACpB,IAAI,QAAQ,UAAU;KACpB,SAAS,QAAQ;KACjB;IACF;IACA,oBAAoB;IACpB,IAAI,oBAAoB;KAKtB,SAAS,QAAQ;KACjB,MAAM,IAAI,wBAAwB,OAAO,WAAW,SAAS;IAC/D;GACF,SAAS,GAAG;IACV,IAAI,aAAa,yBACf,MAAM;IAER,IAAI,EAAE,qBAAqB,GAAG;KAK5B,GAAG,OAAO,yCAAyC,CAAC;KACpD;IACF;IAEA,MAAM;GACR;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,SAAS,GAAG;EACV,IAAI,aAAa,2BAA2B,WAAW,KAAA,GACrD,SAAS;EAEX,MAAM;CACR,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,MAAM,SACtC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAEA,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,QACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;CAQzE,OAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,iCAAiC;EAGlD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,SAAS;IAClE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,6BACF;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAe,WACb,IACA,QACA,oBACA,MACyB;CACzB,IAAI;CACJ,IAAI,CAAC,oBAAoB;EAGvB,MAAM,YAAY,YAAY,IAAI;EAClC,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;EAC7D,8BAA8B,EAAE,SAC9B,YAAY,IAAI,IAAI,WACpB,6BAA6B,QAAQ,MAAM,eAAe,SAAS,CACrE;EACA,GAAG,OAAO,yBAAyB,eAAe,WAAW;EAC7D,qBAAqB;CACvB;CAEA,MAAM,YAAY,gBAAgB,aAAa,OAAO,WAAW;CACjE,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,MAAM,8BAA8B,WAAW,OAAO,QAAQ,IAAI;EAClE,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,SAClB;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE,OAAO;EACX,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EACD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,OAAO,QAAQ;EACjB,GACA;GAAC;GAAK,OAAO;GAAW,aAAa;EAAI,CAC3C;EACA,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM;EACR;EACA,IAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GAAG;GACpC,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,OAAO,QAAQ,MAAM,kBAAkB;EACxE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,OAAO,QAAQ,IAAI;GAChC,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BACH,0BAA0B,EAAE,IAAI,SAAS,OAAO,QAAQ,IAAI,EAAE,MAAM;GAClE,GAAG;GACH,QAAQ;EACV,CAAC;EAEH,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,0BAA0B,EAAE,IAAI,GAAG;GACjC,GAAG;GACH;EACF,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,wBAAwB,IAAI,oBAAoB,EAAE;EAClD,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,mBAAmB,YAAY,gBAAgB;GACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;GACA,OAAO;EACT;EACA,IAAI,YAAY,YAAY,cAAc;GACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;GACA,OAAO;EACT;EACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW,WAAW;CACxE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,WAAW,GAC9C,uBAAuB,IAAI,QAAQ,KAAK,CAC1C,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WAAW,WAC9D;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,OAAO,QAAQ,IAAI,GAAG;EAC7D;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,4BACP,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,iBAAiB,MAAM;CACvB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}

package/out/zero-cache/src/services/litestream/metrics.d.ts

@@ -0,0 +1,30 @@
+import type { ZeroConfig } from '../../config/zero-config.ts';
+export type LitestreamRole = 'replication_manager' | 'view_syncer';
+export type LitestreamVersion = 'legacy' | 'v5';
+export type LitestreamMetricAttrs = {
+    role: LitestreamRole;
+    backup_scheme: string;
+    litestream: LitestreamVersion;
+};
+type LitestreamMultipartMetricAttrs = {
+    multipart_concurrency: number;
+    multipart_size_mib: number;
+};
+export declare function litestreamRestoreMetricAttrs(config: ZeroConfig, role: LitestreamRole, backupURL?: string | undefined): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs;
+export declare function litestreamBackupMetricAttrs(config: ZeroConfig): LitestreamMetricAttrs;
+export declare function litestreamBackupProcessMetricAttrs(config: ZeroConfig): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs;
+export declare function litestreamMonitorMetricAttrs(backupURL: string, litestream: LitestreamVersion, role: LitestreamRole): LitestreamMetricAttrs;
+export declare function litestreamRestoreRuns(): import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
+export declare function litestreamRestoreAttempts(): import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
+export declare function litestreamRestoredDbBytes(): import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
+export declare function litestreamBackupProcessRuns(): import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
+export declare function litestreamRestoreDuration(): import("../../observability/metrics.ts").LatencyHistogram;
+export declare function litestreamRestoreWaitDuration(): import("../../observability/metrics.ts").LatencyHistogram;
+export declare function litestreamRestoreProcessDuration(): import("../../observability/metrics.ts").LatencyHistogram;
+export declare function litestreamRestoreValidationDuration(): import("../../observability/metrics.ts").LatencyHistogram;
+export declare function litestreamBackupProcessDuration(): import("../../observability/metrics.ts").LatencyHistogram;
+export declare function litestreamBackupListDuration(): import("../../observability/metrics.ts").LatencyHistogram;
+export declare function litestreamBackupVerificationDuration(): import("../../observability/metrics.ts").LatencyHistogram;
+export declare function litestreamSnapshotReservationDuration(): import("../../observability/metrics.ts").LatencyHistogram;
+export {};
+//# sourceMappingURL=metrics.d.ts.map

package/out/zero-cache/src/services/litestream/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAM5D,MAAM,MAAM,cAAc,GAAG,qBAAqB,GAAG,aAAa,CAAC;AACnE,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,IAAI,CAAC;AAEhD,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,iBAAiB,CAAC;CAC/B,CAAC;AAEF,KAAK,8BAA8B,GAAG;IACpC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAMF,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,cAAc,EACpB,SAAS,qBAA8B,GACtC,qBAAqB,GAAG,8BAA8B,CAaxD;AAED,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,UAAU,GACjB,qBAAqB,CAUvB;AAED,wBAAgB,kCAAkC,CAChD,MAAM,EAAE,UAAU,GACjB,qBAAqB,GAAG,8BAA8B,CAKxD;AAED,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,iBAAiB,EAC7B,IAAI,EAAE,cAAc,GACnB,qBAAqB,CAMvB;AAyBD,wBAAgB,qBAAqB,kFAMpC;AAED,wBAAgB,yBAAyB,kFAMxC;AAED,wBAAgB,yBAAyB,kFAMxC;AAED,wBAAgB,2BAA2B,kFAM1C;AAED,wBAAgB,yBAAyB,8DAKxC;AAED,wBAAgB,6BAA6B,8DAK5C;AAED,wBAAgB,gCAAgC,8DAK/C;AAED,wBAAgB,mCAAmC,8DAKlD;AAED,wBAAgB,+BAA+B,8DAK9C;AAED,wBAAgB,4BAA4B,8DAK3C;AAED,wBAAgB,oCAAoC,8DAKnD;AAED,wBAAgB,qCAAqC,8DAKpD"}

package/out/zero-cache/src/services/litestream/metrics.js

@@ -0,0 +1,112 @@
+import { getOrCreateCounter, getOrCreateHistogram } from "../../observability/metrics.js";
+//#region ../zero-cache/src/services/litestream/metrics.ts
+var LITESTREAM_DURATION_HISTOGRAM_BOUNDARIES_S = [
+	1,
+	2,
+	5,
+	10,
+	30,
+	60,
+	120,
+	300,
+	600,
+	1200,
+	2400,
+	3600,
+	7200
+];
+function litestreamRestoreMetricAttrs(config, role, backupURL = config.litestream.backupURL) {
+	const { executable, executableV5, restoreUsingV5 } = config.litestream;
+	const selectedExecutable = (restoreUsingV5 ? executableV5 : executable) ?? executable;
+	return {
+		role,
+		backup_scheme: litestreamBackupScheme(backupURL),
+		litestream: executableV5 !== void 0 && selectedExecutable === executableV5 ? "v5" : "legacy",
+		...litestreamMultipartMetricAttrs(config)
+	};
+}
+function litestreamBackupMetricAttrs(config) {
+	return {
+		role: "replication_manager",
+		backup_scheme: litestreamBackupScheme(config.litestream.backupURL),
+		litestream: config.litestream.executableV5 !== void 0 && config.litestream.executable === config.litestream.executableV5 ? "v5" : "legacy"
+	};
+}
+function litestreamBackupProcessMetricAttrs(config) {
+	return {
+		...litestreamBackupMetricAttrs(config),
+		...litestreamMultipartMetricAttrs(config)
+	};
+}
+function litestreamMonitorMetricAttrs(backupURL, litestream, role) {
+	return {
+		role,
+		backup_scheme: litestreamBackupScheme(backupURL),
+		litestream
+	};
+}
+function litestreamBackupScheme(backupURL) {
+	if (!backupURL) return "unknown";
+	try {
+		const protocol = new URL(backupURL).protocol;
+		return protocol.endsWith(":") ? protocol.slice(0, -1) : protocol;
+	} catch {
+		return "unknown";
+	}
+}
+function litestreamMultipartMetricAttrs(config) {
+	return {
+		multipart_concurrency: config.litestream.multipartConcurrency,
+		multipart_size_mib: Math.round(config.litestream.multipartSize / 1024 / 1024)
+	};
+}
+function litestreamRestoreRuns() {
+	return getOrCreateCounter("replica", "litestream_restore_runs", "Litestream restore runs, labeled by result.");
+}
+function litestreamRestoreAttempts() {
+	return getOrCreateCounter("replica", "litestream_restore_attempts", "Litestream restore subprocess attempts, labeled by result.");
+}
+function litestreamRestoredDbBytes() {
+	return getOrCreateCounter("replica", "litestream_restored_db", {
+		description: "SQLite database bytes restored by successful litestream restores.",
+		unit: "bytes"
+	});
+}
+function litestreamBackupProcessRuns() {
+	return getOrCreateCounter("replica", "litestream_backup_process_runs", "Litestream backup process exits, labeled by result.");
+}
+function litestreamRestoreDuration() {
+	return litestreamDurationHistogram("litestream_restore_duration", "Wall-clock duration of a litestream restore run, labeled by result.");
+}
+function litestreamRestoreWaitDuration() {
+	return litestreamDurationHistogram("litestream_restore_wait_duration", "Time spent waiting for the replication-manager snapshot status before restoring.");
+}
+function litestreamRestoreProcessDuration() {
+	return litestreamDurationHistogram("litestream_restore_process_duration", "Wall-clock duration of the litestream restore subprocess.");
+}
+function litestreamRestoreValidationDuration() {
+	return litestreamDurationHistogram("litestream_restore_validation_duration", "Time spent validating a restored replica database.");
+}
+function litestreamBackupProcessDuration() {
+	return litestreamDurationHistogram("litestream_backup_process_duration", "Runtime duration of the litestream backup subprocess before it exits.");
+}
+function litestreamBackupListDuration() {
+	return litestreamDurationHistogram("litestream_backup_list_duration", "Duration of litestream backup destination listing commands.");
+}
+function litestreamBackupVerificationDuration() {
+	return litestreamDurationHistogram("litestream_backup_verification_duration", "Duration of verifying the actual backup state in the backup destination.");
+}
+function litestreamSnapshotReservationDuration() {
+	return litestreamDurationHistogram("litestream_snapshot_reservation_duration", "Duration of a snapshot reservation while a view-syncer restores and subscribes.");
+}
+function litestreamDurationHistogram(name, description) {
+	return getOrCreateHistogram("replica", name, {
+		description,
+		unit: "s",
+		bucketBoundaries: LITESTREAM_DURATION_HISTOGRAM_BOUNDARIES_S
+	});
+}
+//#endregion
+export { litestreamBackupListDuration, litestreamBackupMetricAttrs, litestreamBackupProcessDuration, litestreamBackupProcessMetricAttrs, litestreamBackupProcessRuns, litestreamBackupVerificationDuration, litestreamMonitorMetricAttrs, litestreamRestoreAttempts, litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreProcessDuration, litestreamRestoreRuns, litestreamRestoreValidationDuration, litestreamRestoreWaitDuration, litestreamRestoredDbBytes, litestreamSnapshotReservationDuration };
+
+//# sourceMappingURL=metrics.js.map

package/out/zero-cache/src/services/litestream/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/metrics.ts"],"sourcesContent":["import type {ZeroConfig} from '../../config/zero-config.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateHistogram,\n} from '../../observability/metrics.ts';\n\nexport type LitestreamRole = 'replication_manager' | 'view_syncer';\nexport type LitestreamVersion = 'legacy' | 'v5';\n\nexport type LitestreamMetricAttrs = {\n  role: LitestreamRole;\n  backup_scheme: string;\n  litestream: LitestreamVersion;\n};\n\ntype LitestreamMultipartMetricAttrs = {\n  multipart_concurrency: number;\n  multipart_size_mib: number;\n};\n\nconst LITESTREAM_DURATION_HISTOGRAM_BOUNDARIES_S = [\n  1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200,\n];\n\nexport function litestreamRestoreMetricAttrs(\n  config: ZeroConfig,\n  role: LitestreamRole,\n  backupURL = config.litestream.backupURL,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n  const {executable, executableV5, restoreUsingV5} = config.litestream;\n  const selectedExecutable =\n    (restoreUsingV5 ? executableV5 : executable) ?? executable;\n  return {\n    role,\n    backup_scheme: litestreamBackupScheme(backupURL),\n    litestream:\n      executableV5 !== undefined && selectedExecutable === executableV5\n        ? 'v5'\n        : 'legacy',\n    ...litestreamMultipartMetricAttrs(config),\n  };\n}\n\nexport function litestreamBackupMetricAttrs(\n  config: ZeroConfig,\n): LitestreamMetricAttrs {\n  return {\n    role: 'replication_manager',\n    backup_scheme: litestreamBackupScheme(config.litestream.backupURL),\n    litestream:\n      config.litestream.executableV5 !== undefined &&\n      config.litestream.executable === config.litestream.executableV5\n        ? 'v5'\n        : 'legacy',\n  };\n}\n\nexport function litestreamBackupProcessMetricAttrs(\n  config: ZeroConfig,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n  return {\n    ...litestreamBackupMetricAttrs(config),\n    ...litestreamMultipartMetricAttrs(config),\n  };\n}\n\nexport function litestreamMonitorMetricAttrs(\n  backupURL: string,\n  litestream: LitestreamVersion,\n  role: LitestreamRole,\n): LitestreamMetricAttrs {\n  return {\n    role,\n    backup_scheme: litestreamBackupScheme(backupURL),\n    litestream,\n  };\n}\n\nfunction litestreamBackupScheme(backupURL: string | undefined): string {\n  if (!backupURL) {\n    return 'unknown';\n  }\n  try {\n    const protocol = new URL(backupURL).protocol;\n    return protocol.endsWith(':') ? protocol.slice(0, -1) : protocol;\n  } catch {\n    return 'unknown';\n  }\n}\n\nfunction litestreamMultipartMetricAttrs(\n  config: ZeroConfig,\n): LitestreamMultipartMetricAttrs {\n  return {\n    multipart_concurrency: config.litestream.multipartConcurrency,\n    multipart_size_mib: Math.round(\n      config.litestream.multipartSize / 1024 / 1024,\n    ),\n  };\n}\n\nexport function litestreamRestoreRuns() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream_restore_runs',\n    'Litestream restore runs, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreAttempts() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream_restore_attempts',\n    'Litestream restore subprocess attempts, labeled by result.',\n  );\n}\n\nexport function litestreamRestoredDbBytes() {\n  return getOrCreateCounter('replica', 'litestream_restored_db', {\n    description:\n      'SQLite database bytes restored by successful litestream restores.',\n    unit: 'bytes',\n  });\n}\n\nexport function litestreamBackupProcessRuns() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream_backup_process_runs',\n    'Litestream backup process exits, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreDuration() {\n  return litestreamDurationHistogram(\n    'litestream_restore_duration',\n    'Wall-clock duration of a litestream restore run, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreWaitDuration() {\n  return litestreamDurationHistogram(\n    'litestream_restore_wait_duration',\n    'Time spent waiting for the replication-manager snapshot status before restoring.',\n  );\n}\n\nexport function litestreamRestoreProcessDuration() {\n  return litestreamDurationHistogram(\n    'litestream_restore_process_duration',\n    'Wall-clock duration of the litestream restore subprocess.',\n  );\n}\n\nexport function litestreamRestoreValidationDuration() {\n  return litestreamDurationHistogram(\n    'litestream_restore_validation_duration',\n    'Time spent validating a restored replica database.',\n  );\n}\n\nexport function litestreamBackupProcessDuration() {\n  return litestreamDurationHistogram(\n    'litestream_backup_process_duration',\n    'Runtime duration of the litestream backup subprocess before it exits.',\n  );\n}\n\nexport function litestreamBackupListDuration() {\n  return litestreamDurationHistogram(\n    'litestream_backup_list_duration',\n    'Duration of litestream backup destination listing commands.',\n  );\n}\n\nexport function litestreamBackupVerificationDuration() {\n  return litestreamDurationHistogram(\n    'litestream_backup_verification_duration',\n    'Duration of verifying the actual backup state in the backup destination.',\n  );\n}\n\nexport function litestreamSnapshotReservationDuration() {\n  return litestreamDurationHistogram(\n    'litestream_snapshot_reservation_duration',\n    'Duration of a snapshot reservation while a view-syncer restores and subscribes.',\n  );\n}\n\nfunction litestreamDurationHistogram(name: string, description: string) {\n  return getOrCreateHistogram('replica', name, {\n    description,\n    unit: 's',\n    bucketBoundaries: LITESTREAM_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n"],"mappings":";;AAoBA,IAAM,6CAA6C;CACjD;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAK;CAAM;CAAM;CAAM;AACxD;AAEA,SAAgB,6BACd,QACA,MACA,YAAY,OAAO,WAAW,WAC0B;CACxD,MAAM,EAAC,YAAY,cAAc,mBAAkB,OAAO;CAC1D,MAAM,sBACH,iBAAiB,eAAe,eAAe;CAClD,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C,YACE,iBAAiB,KAAA,KAAa,uBAAuB,eACjD,OACA;EACN,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,4BACd,QACuB;CACvB,OAAO;EACL,MAAM;EACN,eAAe,uBAAuB,OAAO,WAAW,SAAS;EACjE,YACE,OAAO,WAAW,iBAAiB,KAAA,KACnC,OAAO,WAAW,eAAe,OAAO,WAAW,eAC/C,OACA;CACR;AACF;AAEA,SAAgB,mCACd,QACwD;CACxD,OAAO;EACL,GAAG,4BAA4B,MAAM;EACrC,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,6BACd,WACA,YACA,MACuB;CACvB,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C;CACF;AACF;AAEA,SAAS,uBAAuB,WAAuC;CACrE,IAAI,CAAC,WACH,OAAO;CAET,IAAI;EACF,MAAM,WAAW,IAAI,IAAI,SAAS,EAAE;EACpC,OAAO,SAAS,SAAS,GAAG,IAAI,SAAS,MAAM,GAAG,EAAE,IAAI;CAC1D,QAAQ;EACN,OAAO;CACT;AACF;AAEA,SAAS,+BACP,QACgC;CAChC,OAAO;EACL,uBAAuB,OAAO,WAAW;EACzC,oBAAoB,KAAK,MACvB,OAAO,WAAW,gBAAgB,OAAO,IAC3C;CACF;AACF;AAEA,SAAgB,wBAAwB;CACtC,OAAO,mBACL,WACA,2BACA,6CACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBACL,WACA,+BACA,4DACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBAAmB,WAAW,0BAA0B;EAC7D,aACE;EACF,MAAM;CACR,CAAC;AACH;AAEA,SAAgB,8BAA8B;CAC5C,OAAO,mBACL,WACA,kCACA,qDACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,4BACL,+BACA,qEACF;AACF;AAEA,SAAgB,gCAAgC;CAC9C,OAAO,4BACL,oCACA,kFACF;AACF;AAEA,SAAgB,mCAAmC;CACjD,OAAO,4BACL,uCACA,2DACF;AACF;AAEA,SAAgB,sCAAsC;CACpD,OAAO,4BACL,0CACA,oDACF;AACF;AAEA,SAAgB,kCAAkC;CAChD,OAAO,4BACL,sCACA,uEACF;AACF;AAEA,SAAgB,+BAA+B;CAC7C,OAAO,4BACL,mCACA,6DACF;AACF;AAEA,SAAgB,uCAAuC;CACrD,OAAO,4BACL,2CACA,0EACF;AACF;AAEA,SAAgB,wCAAwC;CACtD,OAAO,4BACL,4CACA,iFACF;AACF;AAEA,SAAS,4BAA4B,MAAc,aAAqB;CACtE,OAAO,qBAAqB,WAAW,MAAM;EAC3C;EACA,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH"}