@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.10

package/out/zero-cache/src/services/change-streamer/forwarder.js.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.js","names":["#lc","#progressMonitorOptions","#active","#queued","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#trackProgress","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast} from './broadcast.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer-service.ts';\nimport type {Status} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusPaddingSeconds: number;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #progressMonitorOptions: ProgressMonitorOptions;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {flowControlConsensusPaddingSeconds: 1},\n  ) {\n    this.#lc = lc.withContext('component', 'progress-monitor');\n    this.#progressMonitorOptions = opts;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(this.#trackProgress, 1000);\n  }\n\n  readonly #trackProgress = () => {\n    const now = performance.now();\n    for (const sub of this.#active) {\n      sub.sampleProcessRate(now);\n    }\n\n    const {flowControlConsensusPaddingSeconds} = this.#progressMonitorOptions;\n    // A negative number disables early flow control release.\n    if (flowControlConsensusPaddingSeconds >= 0) {\n      this.#currentBroadcast?.checkProgress(\n        this.#lc,\n        flowControlConsensusPaddingSeconds * 1000,\n        now,\n      );\n    }\n  };\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const start = performance.now();\n    const broadcast = new Broadcast(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAgBA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAAC,oCAAoC,EAAC,GACrE;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKC,0BAA0B;EAE/B,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKG,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,YAAY,KAAKC,gBAAgB,GAAI;CAC/D;CAEA,uBAAgC;EAC9B,MAAM,MAAM,YAAY,IAAI;EAC5B,KAAK,MAAM,OAAO,KAAKN,SACrB,IAAI,kBAAkB,GAAG;EAG3B,MAAM,EAAC,uCAAsC,KAAKD;EAElD,IAAI,sCAAsC,GACxC,KAAKQ,mBAAmB,cACtB,KAAKT,KACL,qCAAqC,KACrC,GACF;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKO,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKG,gBACP,KAAKP,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKS,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKT,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UAAU,KAAKA,QAAQ,OAAO,GAAG,KAAK;EAC5D,KAAKS,yBAAyB,MAAM,EAAE;EAGtC,KAAKF,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKL,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKI,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKP,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKO,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKP,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}
+{"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusTimeoutProportion: number;\n  flowControlSlowSubscriberGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #earlyReleaseOptions: EarlyReleaseOptions | undefined;\n  readonly #flowControlSlowSubscriberGracePeriodMs: number | undefined;\n\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {\n      flowControlConsensusTimeoutProportion: 2.0,\n    },\n  ) {\n    this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    this.#earlyReleaseOptions =\n      opts.flowControlConsensusTimeoutProportion < 0\n        ? undefined\n        : {\n            consensusTimeoutProportion:\n              opts.flowControlConsensusTimeoutProportion,\n            setTimeoutFn: this.#setTimeout,\n            clearTimeoutFn: this.#clearTimeout,\n          };\n    this.#flowControlSlowSubscriberGracePeriodMs =\n      opts.flowControlSlowSubscriberGracePeriodMs;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(\n      () => this.checkSubscriberProgress(performance.now()),\n      1000,\n    );\n  }\n\n  /**\n   * Runs one progress-monitor tick: samples each active subscriber's change\n   * rate, logs slow broadcasts, and disconnects subscribers that have been\n   * continuously lagging for longer than the configured grace period.\n   *\n   * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n   * than private) so tests can drive a single tick with an explicit `now`.\n   */\n  checkSubscriberProgress(now: number) {\n    let timedOut: Map<Subscriber, SubscriberStats> | undefined;\n    let slowestOnTime: SubscriberStats | undefined;\n\n    for (const sub of this.#active) {\n      const stats = sub.sampleProcessRate(now).getStats();\n      if (stats.missedLastTimeout) {\n        (timedOut ??= new Map()).set(sub, stats);\n      } else if (\n        stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n      ) {\n        slowestOnTime = stats;\n      }\n    }\n    this.#currentBroadcast?.logSlowRequests(now);\n\n    if (\n      this.#flowControlSlowSubscriberGracePeriodMs &&\n      timedOut?.size &&\n      slowestOnTime\n    ) {\n      for (const [sub, stats] of timedOut.entries()) {\n        const rate =\n          stats.processRate < slowestOnTime.processRate\n            ? 'lagging'\n            : 'catching-up';\n        const laggingDuration = sub.reportChangeRate(now, rate);\n        if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n          this.#lc.warn?.(\n            `subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n            {stats},\n          );\n          sub.close(\n            StreamTooFarBehind,\n            `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n          );\n        }\n      }\n    }\n  }\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const start = performance.now();\n    const broadcast = new Broadcast(\n      this.#lc,\n      this.#active.values(),\n      entry,\n      this.#earlyReleaseOptions,\n    );\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAoBA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAAyC;IACnE,KAAKN,IAAI,OACP,cAAc,IAAI,GAAG,kBAAkB,gBAAgB,oBACvD,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EACF;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKE,gBACP,KAAKR,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKU,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKV,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKH,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKQ,yBAAyB,MAAM,EAAE;EAGtC,KAAKF,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKR,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKQ,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKR,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/{litestream3-backup-monitor.d.ts → litestream3-prometheus-poller.d.ts}

@@ -1,9 +1,7 @@
 import type { LogContext } from '@rocicorp/logger';
-import { Subscription } from '../../types/subscription.ts';
-import type { BackupMonitor } from './backup-monitor.ts';
-import type { ChangeStreamerService } from './change-streamer.ts';
-import type { SnapshotMessage } from './snapshot.ts';
-export declare const CHECK_INTERVAL_MS = 60000;
+import type { Source } from '../../types/streams.ts';
+import type { BackedUpWatermark } from './backup-monitor.ts';
+export declare const CHECK_INTERVAL_MS = 10000;
 /**
  * Allowance for clock skew between the machine reporting litestream metrics
  * and the timestamps reported by the backup destination (e.g. S3).
@@ -62,53 +60,29 @@ export declare const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10000;
  */
 export type BackupStateVerifier = () => Promise<Date>;
 /**
- * The Litestream3BackupMonitor polls the litestream "/metrics" endpoint to
+ * The Litestream3PrometheusPoller polls the litestream "/metrics" endpoint to
  * track the watermark (label) value of the `litestream_replica_progress` gauge
- * and schedules cleanup of change log entries that can be purged as a result.
+ * and emits verified watermarks to a stream of {@link BackedUpWatermark}s.
  *
  * See: https://github.com/rocicorp/litestream/pull/3
  *
- * Note that change log entries cannot simply be purged as soon as they
- * have been applied and backed up by litestream. Consider the case in which
- * litestream backs up new wal segments every minute, but it takes 5 minutes
- * to restore a replica: if a zero-cache starts restoring a replica at
- * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
- * purging changelog records as soon as those watermarks are replicated would
- * result in the zero-cache not being able to catch up from minute 0 once it
- * has finished restoring the replica.
- *
- * The `/snapshot` reservation protocol is used to prevent premature change
- * log cleanup:
- * - Clients restoring a snapshot initiate a `/snapshot` request and hold that
- *   request open while it restores its snapshot, prepares it, and
- *   starts its subscription to the change stream. During this time, no
- *   cleanups are scheduled.
- * - When the subscription is started, the interval since the beginning of
- *   of the reservation is tracked to increase the background cleanup delay
- *   interval if needed. The reservation is ended (and request closed), and
- *   cleanup scheduling is resumed with the current delay interval.
- *
- * Note that the reservation request is the primary mechanism by which
- * premature change log cleanup is prevented. The cleanup delay interval is
- * a secondary safeguard.
- *
- * Additionally, because the watermarks reported by litestream metrics
- * reflect what litestream *believes* has been backed up (which has been
- * observed to diverge from reality when uploads silently fail), the cleanup
- * watermark is only advanced after verifying it against the actual backup
- * state in the replica destination via a {@link BackupStateVerifier}. If the
+ * Because the watermarks reported by litestream metrics reflect what
+ * litestream *believes* has been backed up (which has been observed to
+ * diverge from reality when uploads silently fail), the cleanup watermark is
+ * only published after verifying it against the actual backup state in the
+ * replica destination via a {@link BackupStateVerifier}. If the
  * actual backup state stays behind the claimed progress for longer than
  * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the
  * process is shut down rather than risk corrupting the backup.
  */
-export declare class Litestream3BackupMonitor implements BackupMonitor {
+export declare class Litestream3PrometheusPoller {
     #private;
-    readonly id = "backup-monitor";
-    constructor(lc: LogContext, replicaFile: string, backupURL: string, metricsEndpoint: string, changeStreamer: ChangeStreamerService, initialCleanupDelayMs: number, verifyBackupState: BackupStateVerifier);
-    run(): Promise<void>;
-    startSnapshotReservation(taskID: string): Subscription<SnapshotMessage>;
-    endReservation(taskID: string, updateCleanupDelay?: boolean): void;
-    readonly checkWatermarksAndScheduleCleanup: () => Promise<void>;
-    stop(): Promise<void>;
+    constructor(lc: LogContext, replicaFile: string, backupURL: string, metricsEndpoint: string, verifyBackupState: BackupStateVerifier);
+    /**
+     * Starts polling for watermarks and pushes new ones to returned stream.
+     * Must only be called once.
+     */
+    start(): Source<BackedUpWatermark>;
+    readonly checkVerifyAndPublishWatermarks: () => Promise<void>;
 }
-//# sourceMappingURL=litestream3-backup-monitor.d.ts.map
+//# sourceMappingURL=litestream3-prometheus-poller.d.ts.map

package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream3-prometheus-poller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAOnD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAE3D,eAAO,MAAM,iBAAiB,QAAS,CAAC;AAExC;;;GAGG;AACH,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,wBAAwB,QAAc,CAAC;AAIpD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAG5D;;;;GAIG;AACH,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAEzD;;;;;;;GAOG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAEtD;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,2BAA2B;;gBAwCpC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,iBAAiB,EAAE,mBAAmB;IAexC;;;OAGG;IACH,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC;IA8BlC,QAAQ,CAAC,+BAA+B,sBAgBtC;CAmSH"}

package/out/zero-cache/src/services/change-streamer/{litestream3-backup-monitor.js → litestream3-prometheus-poller.js}

@@ -1,17 +1,14 @@
-import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
-import { sleep } from "../../../../shared/src/sleep.js";
+import { assert } from "../../../../shared/src/asserts.js";
 import { must } from "../../../../shared/src/must.js";
-import { Database } from "../../../../zqlite/src/db.js";
-import { getOrCreateCounter, getOrCreateGauge } from "../../observability/metrics.js";
+import { getOrCreateCounter } from "../../observability/metrics.js";
 import { RunningState, UnrecoverableError } from "../running-state.js";
+import { versionFromLexi } from "../../types/lexi-version.js";
 import { Subscription } from "../../types/subscription.js";
-import { litestreamBackupVerificationDuration, litestreamMonitorMetricAttrs, litestreamSnapshotReservationDuration } from "../litestream/metrics.js";
-import { resolver } from "@rocicorp/resolver";
+import { litestreamBackupVerificationDuration, litestreamMonitorMetricAttrs } from "../litestream/metrics.js";
 import { statSync } from "node:fs";
 import parsePrometheusTextFormat from "parse-prometheus-text-format";
-//#region ../zero-cache/src/services/change-streamer/litestream3-backup-monitor.ts
-var CHECK_INTERVAL_MS = 6e4;
-var MIN_CLEANUP_DELAY_MS = 3e4;
+//#region ../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts
+var CHECK_INTERVAL_MS = 1e4;
 /**
 * Allowance for clock skew between the machine reporting litestream metrics
 * and the timestamps reported by the backup destination (e.g. S3).
@@ -57,126 +54,58 @@ var GiB = 1024 ** 3;
 var INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 6e4;
 var INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB;
 /**
-* How often the BackupMonitor re-checks whether the first restorable backup has
-* appeared while a view-syncer holds a snapshot reservation open during a cold
-* start.
-*/
-var RESTORABLE_BACKUP_POLL_INTERVAL_MS = 1e4;
-/**
-* The Litestream3BackupMonitor polls the litestream "/metrics" endpoint to
+* The Litestream3PrometheusPoller polls the litestream "/metrics" endpoint to
 * track the watermark (label) value of the `litestream_replica_progress` gauge
-* and schedules cleanup of change log entries that can be purged as a result.
+* and emits verified watermarks to a stream of {@link BackedUpWatermark}s.
 *
 * See: https://github.com/rocicorp/litestream/pull/3
 *
-* Note that change log entries cannot simply be purged as soon as they
-* have been applied and backed up by litestream. Consider the case in which
-* litestream backs up new wal segments every minute, but it takes 5 minutes
-* to restore a replica: if a zero-cache starts restoring a replica at
-* minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
-* purging changelog records as soon as those watermarks are replicated would
-* result in the zero-cache not being able to catch up from minute 0 once it
-* has finished restoring the replica.
-*
-* The `/snapshot` reservation protocol is used to prevent premature change
-* log cleanup:
-* - Clients restoring a snapshot initiate a `/snapshot` request and hold that
-*   request open while it restores its snapshot, prepares it, and
-*   starts its subscription to the change stream. During this time, no
-*   cleanups are scheduled.
-* - When the subscription is started, the interval since the beginning of
-*   of the reservation is tracked to increase the background cleanup delay
-*   interval if needed. The reservation is ended (and request closed), and
-*   cleanup scheduling is resumed with the current delay interval.
-*
-* Note that the reservation request is the primary mechanism by which
-* premature change log cleanup is prevented. The cleanup delay interval is
-* a secondary safeguard.
-*
-* Additionally, because the watermarks reported by litestream metrics
-* reflect what litestream *believes* has been backed up (which has been
-* observed to diverge from reality when uploads silently fail), the cleanup
-* watermark is only advanced after verifying it against the actual backup
-* state in the replica destination via a {@link BackupStateVerifier}. If the
+* Because the watermarks reported by litestream metrics reflect what
+* litestream *believes* has been backed up (which has been observed to
+* diverge from reality when uploads silently fail), the cleanup watermark is
+* only published after verifying it against the actual backup state in the
+* replica destination via a {@link BackupStateVerifier}. If the
 * actual backup state stays behind the claimed progress for longer than
 * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the
 * process is shut down rather than risk corrupting the backup.
 */
-var Litestream3BackupMonitor = class {
-	id = "backup-monitor";
+var Litestream3PrometheusPoller = class {
 	#lc;
 	#replicaFile;
 	#backupURL;
 	#metricsEndpoint;
-	#changeStreamer;
-	#state = new RunningState(this.id);
-	#reservations = /* @__PURE__ */ new Map();
+	#state = new RunningState("litestream3-prometheus-poller");
+	#stream;
 	#watermarks = /* @__PURE__ */ new Map();
 	#verifyBackupState;
 	#purgesBlocked = getOrCreateCounter("replica", "purge_blocked", { description: "Number of change-log purges blocked because the actual backup state (as listed from the replica destination) could not be verified, or is older than the backup progress claimed by litestream metrics. A steadily increasing value indicates a wedged or failing backup. The \"backup-wedged\" reason is emitted once just before the process shuts itself down due to a persistently stale backup." });
-	#fatal = resolver();
 	#lastWatermark = "";
 	#latestBackupTime = null;
 	#lastVerifiedUploadTime = null;
 	#backupStaleSince = null;
 	#runStartTime = null;
-	#cleanupDelayMs;
 	#checkMetricsTimer;
-	constructor(lc, replicaFile, backupURL, metricsEndpoint, changeStreamer, initialCleanupDelayMs, verifyBackupState) {
-		this.#lc = lc.withContext("component", this.id);
+	constructor(lc, replicaFile, backupURL, metricsEndpoint, verifyBackupState) {
+		this.#lc = lc.withContext("component", "litestream3-prometheus-poller");
 		this.#replicaFile = replicaFile;
 		this.#backupURL = backupURL;
 		this.#metricsEndpoint = metricsEndpoint;
-		this.#changeStreamer = changeStreamer;
 		this.#verifyBackupState = verifyBackupState;
-		this.#cleanupDelayMs = Math.max(initialCleanupDelayMs, MIN_CLEANUP_DELAY_MS);
-		this.#lc.info?.(`backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`);
-	}
-	run() {
-		this.#runStartTime = Date.now();
-		this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint} with ${this.#cleanupDelayMs} ms cleanup delay`);
-		this.#checkMetricsTimer = setInterval(this.checkWatermarksAndScheduleCleanup, CHECK_INTERVAL_MS);
-		this.#initBackupLagMetric();
-		return Promise.race([this.#state.stopped(), this.#fatal.promise]);
-	}
-	startSnapshotReservation(taskID) {
-		this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);
-		this.#reservations.get(taskID)?.sub.cancel();
-		const sub = Subscription.create({ cleanup: () => this.endReservation(taskID, false) });
-		this.#reservations.set(taskID, {
-			start: /* @__PURE__ */ new Date(),
-			sub
-		});
-		this.#pushStatusWhenRestorable(taskID, sub);
-		return sub;
+		this.#stream = Subscription.create({ cleanup: () => {
+			clearInterval(this.#checkMetricsTimer);
+			this.#state.stop(this.#lc);
+		} });
 	}
 	/**
-	* Pushes the `status` snapshot signal once a restorable backup is confirmed
-	* to exist in the backup destination. On a warm start this is immediate; on a
-	* cold start (a fresh stack whose first backup is still being produced) the
-	* reservation is held open — kept alive by websocket liveness pings — and the
-	* view-syncer stays unready until the first backup lands, rather than being
-	* told to restore a backup that does not yet exist.
+	* Starts polling for watermarks and pushes new ones to returned stream.
+	* Must only be called once.
 	*/
-	async #pushStatusWhenRestorable(taskID, sub) {
-		try {
-			while (this.#lastVerifiedUploadTime === null) {
-				if (!sub.active) return;
-				if (await this.#confirmRestorableBackup()) break;
-				this.#lc.info?.(`no restorable backup at ${this.#backupURL} yet; holding ${taskID}'s reservation open until it lands`);
-				await sleep(RESTORABLE_BACKUP_POLL_INTERVAL_MS, this.#state.signal);
-			}
-			const changeLogState = await this.#changeStreamer.getChangeLogState();
-			if (sub.active) sub.push(["status", {
-				tag: "status",
-				backupURL: this.#backupURL,
-				...changeLogState
-			}]);
-		} catch (e) {
-			if (this.#state.signal.aborted) return;
-			this.#lc.warn?.(`failing snapshot reservation`, e);
-			sub.fail(e instanceof Error ? e : new Error(String(e)));
-		}
+	start() {
+		assert(this.#runStartTime === null, `Already called start()`);
+		this.#runStartTime = Date.now();
+		this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint}`);
+		this.#checkMetricsTimer = setInterval(this.checkVerifyAndPublishWatermarks, CHECK_INTERVAL_MS);
+		return this.#stream;
 	}
 	/**
 	* Verifies whether a restorable backup actually exists in the destination
@@ -195,33 +124,14 @@ var Litestream3BackupMonitor = class {
 			return false;
 		}
 	}
-	endReservation(taskID, updateCleanupDelay = true) {
-		const res = this.#reservations.get(taskID);
-		if (res === void 0) return;
-		this.#reservations.delete(taskID);
-		const { start, sub } = res;
-		sub.cancel();
-		const duration = Date.now() - start.getTime();
-		litestreamSnapshotReservationDuration().recordMs(duration, {
-			...litestreamMonitorMetricAttrs(this.#backupURL, "legacy", "view_syncer"),
-			result: updateCleanupDelay ? "success" : "cancelled"
-		});
-		if (updateCleanupDelay) {
-			this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);
-			if (duration > this.#cleanupDelayMs) {
-				this.#cleanupDelayMs = duration;
-				this.#lc.info?.(`increased cleanup delay to ${duration} ms`);
-			}
-		}
-	}
-	checkWatermarksAndScheduleCleanup = async () => {
+	checkVerifyAndPublishWatermarks = async () => {
 		try {
 			await this.#checkWatermarks();
 		} catch (e) {
 			this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);
 		}
 		try {
-			await this.#scheduleCleanup();
+			await this.#publishVerifiedWatermarks();
 		} catch (e) {
 			this.#lc.warn?.(`error scheduling cleanup`, e);
 		}
@@ -251,11 +161,16 @@ var Litestream3BackupMonitor = class {
 			const watermark = metric.labels?.watermark;
 			const name = metric.labels?.name;
 			const time = /* @__PURE__ */ new Date(parseFloat(metric.value) * 1e3);
-			if (watermark) yield {
-				watermark,
-				time,
-				name
-			};
+			if (watermark) try {
+				versionFromLexi(watermark);
+				yield {
+					watermark,
+					time,
+					name
+				};
+			} catch (e) {
+				this.#lc.warn?.(`ignoring invalid watermark ${watermark}`, { metric }, e);
+			}
 		}
 	}
 	async #checkWatermarks() {
@@ -266,15 +181,10 @@ var Litestream3BackupMonitor = class {
 		}
 		return this.#latestBackupTime;
 	}
-	async #scheduleCleanup() {
-		if (this.#reservations.size > 0) {
-			this.#lc.info?.(`watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`);
-			return;
-		}
-		const latestCleanupTime = Date.now() - this.#cleanupDelayMs;
-		const maxWatermark = this.#maxWatermarkUpTo(latestCleanupTime);
-		if (maxWatermark.length === 0) return;
-		const claimedTime = must(this.#watermarks.get(maxWatermark));
+	async #publishVerifiedWatermarks() {
+		const maxBackup = this.#maxBackupBefore(Date.now());
+		if (!maxBackup) return;
+		const { watermark: maxWatermark, backupTime: claimedTime } = maxBackup;
 		if (!this.#confirmedDurable(claimedTime)) try {
 			this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();
 		} catch (e) {
@@ -283,8 +193,8 @@ var Litestream3BackupMonitor = class {
 			return;
 		}
 		const lastUpload = must(this.#lastVerifiedUploadTime);
-		const verifiedWatermark = this.#maxWatermarkUpTo(Math.min(latestCleanupTime, lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS));
-		if (verifiedWatermark.length === 0) {
+		const verifiedBackup = this.#maxBackupBefore(lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS);
+		if (!verifiedBackup) {
 			this.#purgesBlocked.add(1, { reason: "backup-stale" });
 			const now = Date.now();
 			if (this.#backupStaleSince === null) this.#backupStaleSince = now;
@@ -294,8 +204,12 @@ var Litestream3BackupMonitor = class {
 			return;
 		}
 		this.#backupStaleSince = null;
-		this.#changeStreamer.scheduleCleanup(verifiedWatermark);
+		const { watermark: verifiedWatermark, backupTime } = verifiedBackup;
 		for (const watermark of this.#watermarks.keys()) if (watermark <= verifiedWatermark) this.#watermarks.delete(watermark);
+		this.#stream.push({
+			watermark: verifiedWatermark,
+			backupTimeMs: backupTime.getTime()
+		});
 		this.#lastWatermark = verifiedWatermark;
 	}
 	async #verifyBackupStateTimed() {
@@ -324,8 +238,7 @@ var Litestream3BackupMonitor = class {
 		this.#purgesBlocked.add(1, { reason: "backup-wedged" });
 		const err = new UnrecoverableError(`backup at ${this.#backupURL} is wedged: litestream claims a backup at ${claimedTime.toISOString()}, but the last object actually uploaded was at ${lastUpload.toISOString()}, and the backup has not advanced for ${staleForMs} ms (grace period ${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting the backup; investigate the backup destination.`);
 		this.#lc.error?.(err.message);
-		clearInterval(this.#checkMetricsTimer);
-		this.#fatal.reject(err);
+		this.#stream.fail(err);
 	}
 	/**
 	* On a fresh stack, fails loudly if no restorable backup has appeared within
@@ -365,17 +278,24 @@ var Litestream3BackupMonitor = class {
 	#shutDownOnMissingInitialBackup(elapsedMs) {
 		const err = new UnrecoverableError(`no restorable backup has appeared at ${this.#backupURL} within ${elapsedMs} ms of startup (grace period ${this.#initialBackupGraceMs()} ms). The initial backup pipeline appears to be broken; shutting down so that the replication-manager is flagged as the cause rather than the view-syncers waiting on it. Investigate litestream replication to ${this.#backupURL}.`);
 		this.#lc.error?.(err.message);
-		clearInterval(this.#checkMetricsTimer);
-		this.#fatal.reject(err);
+		this.#stream.fail(err);
 	}
 	/**
 	* Returns the newest watermark whose backup time is at or before `cutoff`
 	* (epoch ms), or `''` if there is none.
 	*/
-	#maxWatermarkUpTo(cutoff) {
+	#maxBackupBefore(cutoff) {
 		let max = "";
-		for (const [watermark, backupTime] of this.#watermarks) if (backupTime.getTime() <= cutoff && watermark > max) max = watermark;
-		return max;
+		let maxBackupTime;
+		for (const [watermark, backupTime] of this.#watermarks) if (backupTime.getTime() <= cutoff && watermark > max) {
+			max = watermark;
+			maxBackupTime = backupTime;
+		}
+		if (!maxBackupTime) return null;
+		return {
+			watermark: max,
+			backupTime: maxBackupTime
+		};
 	}
 	/**
 	* Returns `true` if the actual backup state, as last verified against the
@@ -388,36 +308,8 @@ var Litestream3BackupMonitor = class {
 		const lastUpload = this.#lastVerifiedUploadTime;
 		return lastUpload !== null && claimedTime.getTime() <= lastUpload.getTime() + 6e4;
 	}
-	stop() {
-		clearInterval(this.#checkMetricsTimer);
-		for (const { sub } of this.#reservations.values()) sub.cancel();
-		this.#state.stop(this.#lc);
-		return promiseVoid;
-	}
-	#initBackupLagMetric() {
-		getOrCreateGauge("replica", "backup_lag", {
-			description: "Latency from when a change is written to the replica to when it is backed up to litestream. It is expected to create a saw pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.",
-			unit: "millisecond"
-		}).addCallback(async (o) => {
-			const latestBackup = await this.#checkWatermarks();
-			if (!latestBackup) {
-				this.#lc.warn?.(`no backed up watermarks. unable to report replica.backup_lag`);
-				return;
-			}
-			const db = new Database(this.#lc, this.#replicaFile, { readonly: true });
-			try {
-				const { writeTimeMs } = db.prepare(`SELECT writeTimeMs FROM "_zero.replicationState"`).get();
-				const backupLag = Math.max(0, writeTimeMs - latestBackup.getTime());
-				o.observe(backupLag);
-			} catch (e) {
-				this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);
-			} finally {
-				db.close();
-			}
-		});
-	}
 };
 //#endregion
-export { Litestream3BackupMonitor };
+export { Litestream3PrometheusPoller };
 
-//# sourceMappingURL=litestream3-backup-monitor.js.map
+//# sourceMappingURL=litestream3-prometheus-poller.js.map

package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream3-prometheus-poller.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#state","#stream","#watermarks","#verifyBackupState","#purgesBlocked","#checkMetricsTimer","#runStartTime","#confirmRestorableBackup","#lastVerifiedUploadTime","#verifyBackupStateTimed","#checkWatermarks","#publishVerifiedWatermarks","#checkInitialBackupDeadline","#fetchWatermarks","#lastWatermark","#latestBackupTime","#maxBackupBefore","#confirmedDurable","#backupStaleSince","#shutDownOnWedgedBackup","#initialBackupGraceMs","#shutDownOnMissingInitialBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamBackupVerificationDuration,\n  litestreamMonitorMetricAttrs,\n} from '../litestream/metrics.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nexport const CHECK_INTERVAL_MS = 10_000;\n\n/**\n * Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n */\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/**\n * How long the actual backup state may remain *continuously* behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n *\n * A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes *some* such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n *\n * The change-log is conservatively *not* purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n */\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 * 60_000; // 15 minutes\n\nconst GiB = 1024 ** 3;\n\n/**\n * How long the BackupMonitor waits for the *first* restorable backup to appear\n * before concluding that the backup pipeline is broken and shutting the process\n * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).\n *\n * On a fresh stack the first backup is not restorable until the initial\n * Postgres->replica sync completes and litestream finishes uploading the\n * initial snapshot — a multipart upload that is not listable in the destination\n * until it is committed. The producer cannot observe that sub-snapshot progress,\n * so this is necessarily a generous timeout. It is scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance (~1 hour per\n * 50GB) so that it never fires before the platform would restart a view-syncer\n * that is waiting on the backup.\n */\nexport const INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 60_000; // 1 hour ...\nconst INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB; // ... per 50 GiB\n\n/**\n * How often the BackupMonitor re-checks whether the first restorable backup has\n * appeared while a view-syncer holds a snapshot reservation open during a cold\n * start.\n */\nexport const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10_000;\n\n/**\n * Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n *\n * See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n */\nexport type BackupStateVerifier = () => Promise<Date>;\n\n/**\n * The Litestream3PrometheusPoller polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and emits verified watermarks to a stream of {@link BackedUpWatermark}s.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Because the watermarks reported by litestream metrics reflect what\n * litestream *believes* has been backed up (which has been observed to\n * diverge from reality when uploads silently fail), the cleanup watermark is\n * only published after verifying it against the actual backup state in the\n * replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n */\nexport class Litestream3PrometheusPoller {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #state = new RunningState('litestream3-prometheus-poller');\n\n  readonly #stream: Subscription<BackedUpWatermark>;\n  readonly #watermarks = new Map<string, Date>();\n\n  readonly #verifyBackupState: BackupStateVerifier;\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup state ' +\n      '(as listed from the replica destination) could not be verified, or ' +\n      'is older than the backup progress claimed by litestream metrics. ' +\n      'A steadily increasing value indicates a wedged or failing backup. ' +\n      'The \"backup-wedged\" reason is emitted once just before the process ' +\n      'shuts itself down due to a persistently stale backup.',\n  });\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  // The time of the most recent object actually verified in the backup\n  // destination, or `null` if none has ever been verified. Only assigned from a\n  // successful verifyBackupState() and never cleared, so a non-null value also\n  // means a restorable backup has been confirmed to exist (which is what gates\n  // the snapshot `status` signal and satisfies the initial-backup deadline).\n  #lastVerifiedUploadTime: Date | null = null;\n  // Epoch ms at which the actual backup state was first observed to be\n  // continuously behind the watermark litestream claims to have backed up,\n  // or `null` while the backup is keeping up. Reset whenever a purge is\n  // confirmed against a real upload.\n  #backupStaleSince: number | null = null;\n  // Epoch ms at which run() started, used to bound how long to wait for the\n  // first restorable backup to appear. `null` until run() is called.\n  #runStartTime: number | null = null;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    verifyBackupState: BackupStateVerifier,\n  ) {\n    this.#lc = lc.withContext('component', 'litestream3-prometheus-poller');\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#verifyBackupState = verifyBackupState;\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        clearInterval(this.#checkMetricsTimer);\n        this.#state.stop(this.#lc);\n      },\n    });\n  }\n\n  /**\n   * Starts polling for watermarks and pushes new ones to returned stream.\n   * Must only be called once.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#runStartTime === null, `Already called start()`);\n    this.#runStartTime = Date.now();\n    this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint}`);\n    this.#checkMetricsTimer = setInterval(\n      this.checkVerifyAndPublishWatermarks,\n      CHECK_INTERVAL_MS,\n    );\n    return this.#stream;\n  }\n\n  /**\n   * Verifies whether a restorable backup actually exists in the destination\n   * (as opposed to what litestream metrics merely claim). On success the\n   * verified upload time is cached — seeding the cleanup-verification fast path\n   * ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.\n   * Returns false if no backup is listable yet (e.g. the initial snapshot is\n   * still uploading).\n   */\n  async #confirmRestorableBackup(): Promise<boolean> {\n    try {\n      this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      return true;\n    } catch (e) {\n      this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);\n      return false;\n    }\n  }\n\n  // Exported for testing\n  readonly checkVerifyAndPublishWatermarks = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      await this.#publishVerifiedWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n    try {\n      await this.#checkInitialBackupDeadline();\n    } catch (e) {\n      this.#lc.warn?.(`error checking initial backup deadline`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n          if (watermark) {\n            try {\n              versionFromLexi(watermark); // Sanity check: Ensure that it is valid.\n              yield {watermark, time, name};\n            } catch (e) {\n              this.#lc.warn?.(\n                `ignoring invalid watermark ${watermark}`,\n                {metric},\n                e,\n              );\n            }\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  async #publishVerifiedWatermarks() {\n    const maxBackup = this.#maxBackupBefore(Date.now());\n    if (!maxBackup) {\n      return;\n    }\n    const {watermark: maxWatermark, backupTime: claimedTime} = maxBackup;\n    // Purge guard: the watermarks (and their backup times) come from\n    // litestream metrics, which are exported when litestream *believes*\n    // an upload succeeded, and have been observed to advance even when\n    // nothing was actually written to the backup destination. Purging the\n    // change-log based on a falsely advancing watermark permanently breaks\n    // the ability to restore + catch up. Before advancing the cleanup\n    // watermark, verify it against the actual backup state: a claimed\n    // backup time is only trusted if an object was actually uploaded to\n    // the replica destination at (or after) that time, modulo clock skew.\n    if (!this.#confirmedDurable(claimedTime)) {\n      try {\n        this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n      } catch (e) {\n        this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n        // Skipping the purge is safe: the change-log just grows.\n        this.#lc.warn?.(\n          `unable to verify backup state. skipping change-log cleanup ` +\n            `up to watermark ${maxWatermark} ` +\n            `(claimed backup time ${claimedTime.toISOString()})`,\n          e,\n        );\n        return;\n      }\n    }\n    // Watermarks whose backup time isn't yet confirmed durable remain in the\n    // map and are re-evaluated at the next check.\n    const lastUpload = must(this.#lastVerifiedUploadTime);\n    const verifiedBackup = this.#maxBackupBefore(\n      lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n    );\n    if (!verifiedBackup) {\n      this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n      const now = Date.now();\n      if (this.#backupStaleSince === null) {\n        this.#backupStaleSince = now;\n      }\n      const staleForMs = now - this.#backupStaleSince;\n      this.#lc.warn?.(\n        `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n          `litestream claims it was backed up at ` +\n          `${claimedTime.toISOString()}, but the last object actually ` +\n          `uploaded to ${this.#backupURL} was at ` +\n          `${lastUpload.toISOString()}. ` +\n          `The backup has been stale for ${staleForMs} ms.`,\n      );\n      if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n        this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n      }\n      return;\n    }\n    // The cleanup watermark advanced past a real upload, so the backup is\n    // keeping up: clear the staleness clock.\n    this.#backupStaleSince = null;\n    const {watermark: verifiedWatermark, backupTime} = verifiedBackup;\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= verifiedWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#stream.push({\n      watermark: verifiedWatermark,\n      backupTimeMs: backupTime.getTime(),\n    });\n    this.#lastWatermark = verifiedWatermark;\n  }\n\n  async #verifyBackupStateTimed(): Promise<Date> {\n    const start = performance.now();\n    let result: 'success' | 'error' = 'error';\n    try {\n      const ret = await this.#verifyBackupState();\n      result = 'success';\n      return ret;\n    } finally {\n      litestreamBackupVerificationDuration().recordMs(\n        performance.now() - start,\n        {\n          ...litestreamMonitorMetricAttrs(\n            this.#backupURL,\n            'legacy',\n            'replication_manager',\n          ),\n          result,\n        },\n      );\n    }\n  }\n\n  /**\n   * Called when the backup has been continuously stale for longer than\n   * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n   * backup, so the process is shut down by rejecting the {@link run()}\n   * promise. The exit is non-zero and logged at ERROR for alerting; on\n   * restart the monitor re-verifies, so a still-wedged backup keeps the\n   * process down until the destination is fixed.\n   */\n  #shutDownOnWedgedBackup(\n    staleForMs: number,\n    claimedTime: Date,\n    lastUpload: Date,\n  ) {\n    this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n    const err = new UnrecoverableError(\n      `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n        `at ${claimedTime.toISOString()}, but the last object actually ` +\n        `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n        `not advanced for ${staleForMs} ms (grace period ` +\n        `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n        `the backup; investigate the backup destination.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * On a fresh stack, fails loudly if no restorable backup has appeared within\n   * a generous, replica-size-scaled grace period (see\n   * {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the \"give up\" decision\n   * onto the replication-manager — the producer responsible for creating the\n   * backup — instead of leaving view-syncers to wait (and the platform to keep\n   * restarting them) while the real fault is here. Does nothing once a\n   * restorable backup has been confirmed.\n   */\n  async #checkInitialBackupDeadline() {\n    if (this.#lastVerifiedUploadTime !== null || this.#runStartTime === null) {\n      return;\n    }\n    const elapsed = Date.now() - this.#runStartTime;\n    if (elapsed < this.#initialBackupGraceMs()) {\n      return;\n    }\n    // The deadline has elapsed without a confirmed backup. Do one definitive\n    // check against the destination before giving up, in case a backup landed\n    // but was never exercised by a reservation or a cleanup verification.\n    if (await this.#confirmRestorableBackup()) {\n      return;\n    }\n    this.#shutDownOnMissingInitialBackup(elapsed);\n  }\n\n  /**\n   * The grace period for the first restorable backup, scaled by replica size to\n   * mirror the platform's storage-scaled startup-probe allowance. Falls back to\n   * the minimum (one unit) if the replica file size cannot be determined.\n   */\n  #initialBackupGraceMs(): number {\n    let bytes = 0;\n    try {\n      bytes = statSync(this.#replicaFile).size;\n    } catch {\n      // Replica file not present/readable yet; fall back to the minimum grace.\n    }\n    const units = Math.max(\n      1,\n      Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES),\n    );\n    return units * INITIAL_BACKUP_GRACE_MS_PER_UNIT;\n  }\n\n  /**\n   * Called when no restorable backup has appeared within\n   * {@link #initialBackupGraceMs}. Continuing to run is pointless — no\n   * view-syncer can restore — and the fault is the backup pipeline, so the\n   * process exits non-zero with a logged error for alerting, mirroring\n   * {@link #shutDownOnWedgedBackup}.\n   */\n  #shutDownOnMissingInitialBackup(elapsedMs: number) {\n    const err = new UnrecoverableError(\n      `no restorable backup has appeared at ${this.#backupURL} within ` +\n        `${elapsedMs} ms of startup (grace period ` +\n        `${this.#initialBackupGraceMs()} ms). The initial backup pipeline ` +\n        `appears to be broken; shutting down so that the replication-manager ` +\n        `is flagged as the cause rather than the view-syncers waiting on it. ` +\n        `Investigate litestream replication to ${this.#backupURL}.`,\n    );\n    this.#lc.error?.(err.message);\n    this.#stream.fail(err);\n  }\n\n  /**\n   * Returns the newest watermark whose backup time is at or before `cutoff`\n   * (epoch ms), or `''` if there is none.\n   */\n  #maxBackupBefore(\n    cutoff: number,\n  ): {watermark: string; backupTime: Date} | null {\n    let max = '';\n    let maxBackupTime: Date | undefined;\n    for (const [watermark, backupTime] of this.#watermarks) {\n      if (backupTime.getTime() <= cutoff && watermark > max) {\n        max = watermark;\n        maxBackupTime = backupTime;\n      }\n    }\n    if (!maxBackupTime) {\n      return null;\n    }\n    return {watermark: max, backupTime: maxBackupTime};\n  }\n\n  /**\n   * Returns `true` if the actual backup state, as last verified against the\n   * backup destination, confirms that data claimed to be backed up at\n   * `claimedTime` is durable (i.e. an object was actually uploaded at or\n   * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n   * clock skew).\n   */\n  #confirmedDurable(claimedTime: Date): boolean {\n    const lastUpload = this.#lastVerifiedUploadTime;\n    return (\n      lastUpload !== null &&\n      claimedTime.getTime() <=\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n    );\n  }\n}\n"],"mappings":";;;;;;;;;;AAgBA,IAAa,oBAAoB;;;;;AAMjC,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B,KAAK;AAE7C,IAAM,MAAM,QAAQ;;;;;;;;;;;;;;;AAgBpB,IAAa,mCAAmC,KAAK;AACrD,IAAM,kCAAkC,KAAK;;;;;;;;;;;;;;;;;AAmC7C,IAAa,8BAAb,MAAyC;CACvC;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,+BAA+B;CAElE;CACA,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAED,iBAAyB;CACzB,oBAAiC;CAMjC,0BAAuC;CAKvC,oBAAmC;CAGnC,gBAA+B;CAC/B;CAEA,YACE,IACA,aACA,WACA,iBACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,+BAA+B;EACtE,KAAKC,eAAe;EACpB,KAAKC,aAAa;EAClB,KAAKC,mBAAmB;EACxB,KAAKI,qBAAqB;EAC1B,KAAKF,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAKI,kBAAkB;GACrC,KAAKL,OAAO,KAAK,KAAKJ,GAAG;EAC3B,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKU,kBAAkB,MAAM,wBAAwB;EAC5D,KAAKA,gBAAgB,KAAK,IAAI;EAC9B,KAAKV,IAAI,OAAO,yBAAyB,KAAKG,kBAAkB;EAChE,KAAKM,qBAAqB,YACxB,KAAK,iCACL,iBACF;EACA,OAAO,KAAKJ;CACd;;;;;;;;;CAUA,MAAMM,2BAA6C;EACjD,IAAI;GACF,KAAKC,0BAA0B,MAAM,KAAKC,wBAAwB;GAClE,OAAO;EACT,SAAS,GAAG;GACV,KAAKb,IAAI,OAAO,gCAAgC,KAAKE,cAAc,CAAC;GACpE,OAAO;EACT;CACF;CAGA,kCAA2C,YAAY;EACrD,IAAI;GACF,MAAM,KAAKY,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKd,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAKY,2BAA2B;EACxC,SAAS,GAAG;GACV,KAAKf,IAAI,OAAO,4BAA4B,CAAC;EAC/C;EACA,IAAI;GACF,MAAM,KAAKgB,4BAA4B;EACzC,SAAS,GAAG;GACV,KAAKhB,IAAI,OAAO,0CAA0C,CAAC;EAC7D;CACF;CAEA,OAAOiB,mBAIJ;EACD,MAAM,kBAAkB,KAAKd;EAC7B,MAAM,SAAS,KAAKC,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAKJ,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAKH,IAAI,OACP,8BAA8B,KAAKG,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GACrD,IAAI,WACF,IAAI;IACF,gBAAgB,SAAS;IACzB,MAAM;KAAC;KAAW;KAAM;IAAI;GAC9B,SAAS,GAAG;IACV,KAAKH,IAAI,OACP,8BAA8B,aAC9B,EAAC,OAAM,GACP,CACF;GACF;EAEJ;CAGN;CAEA,MAAMc,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAKG,iBAAiB,GAChE,IAAI,YAAY,KAAKC,kBAAkB,CAAC,KAAKZ,YAAY,IAAI,SAAS,GAAG;GACvE,KAAKN,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAKM,YAAY,IAAI,WAAW,IAAI;GACpC,KAAKa,oBAAoB;EAC3B;EAEF,OAAO,KAAKA;CACd;CAEA,MAAMJ,6BAA6B;EACjC,MAAM,YAAY,KAAKK,iBAAiB,KAAK,IAAI,CAAC;EAClD,IAAI,CAAC,WACH;EAEF,MAAM,EAAC,WAAW,cAAc,YAAY,gBAAe;EAU3D,IAAI,CAAC,KAAKC,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAKT,0BAA0B,MAAM,KAAKC,wBAAwB;EACpE,SAAS,GAAG;GACV,KAAKL,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAKR,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAKY,uBAAuB;EACpD,MAAM,iBAAiB,KAAKQ,iBAC1B,WAAW,QAAQ,IAAI,4BACzB;EACA,IAAI,CAAC,gBAAgB;GACnB,KAAKZ,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAKc,sBAAsB,MAC7B,KAAKA,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAKA;GAC9B,KAAKtB,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAKE,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAKqB,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAKD,oBAAoB;EACzB,MAAM,EAAC,WAAW,mBAAmB,eAAc;EACnD,KAAK,MAAM,aAAa,KAAKhB,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKD,QAAQ,KAAK;GAChB,WAAW;GACX,cAAc,WAAW,QAAQ;EACnC,CAAC;EACD,KAAKa,iBAAiB;CACxB;CAEA,MAAML,0BAAyC;EAC7C,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAKN,mBAAmB;GAC1C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,EAAE,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAKL,YACL,UACA,qBACF;IACA;GACF,CACF;EACF;CACF;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAKM,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAKN,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;;;;;;CAWA,MAAMW,8BAA8B;EAClC,IAAI,KAAKJ,4BAA4B,QAAQ,KAAKF,kBAAkB,MAClE;EAEF,MAAM,UAAU,KAAK,IAAI,IAAI,KAAKA;EAClC,IAAI,UAAU,KAAKc,sBAAsB,GACvC;EAKF,IAAI,MAAM,KAAKb,yBAAyB,GACtC;EAEF,KAAKc,gCAAgC,OAAO;CAC9C;;;;;;CAOA,wBAAgC;EAC9B,IAAI,QAAQ;EACZ,IAAI;GACF,QAAQ,SAAS,KAAKxB,YAAY,EAAE;EACtC,QAAQ,CAER;EAKA,OAJc,KAAK,IACjB,GACA,KAAK,KAAK,QAAQ,+BAA+B,CAE5C,IAAQ;CACjB;;;;;;;;CASA,gCAAgC,WAAmB;EACjD,MAAM,MAAM,IAAI,mBACd,wCAAwC,KAAKC,WAAW,UACnD,UAAU,+BACV,KAAKsB,sBAAsB,EAAE,kNAGS,KAAKtB,WAAW,EAC7D;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;CAMA,iBACE,QAC8C;EAC9C,IAAI,MAAM;EACV,IAAI;EACJ,KAAK,MAAM,CAAC,WAAW,eAAe,KAAKC,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAAK;GACrD,MAAM;GACN,gBAAgB;EAClB;EAEF,IAAI,CAAC,eACH,OAAO;EAET,OAAO;GAAC,WAAW;GAAK,YAAY;EAAa;CACnD;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAKM;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;AACF"}

package/out/zero-cache/src/services/change-streamer/replica-poller.d.ts

@@ -0,0 +1,15 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Source } from '../../types/streams.ts';
+import type { BackedUpWatermark } from './backup-monitor.ts';
+/**
+ * The single-node equivalent of a backup monitor polls the replica file every
+ * 30 seconds and schedules cleanup when the watermark
+ * (i.e. stateVersion) moves forward.
+ */
+export declare class ReplicaPoller {
+    #private;
+    readonly id = "replica-poller";
+    constructor(lc: LogContext, replicaFile: string);
+    start(): Source<BackedUpWatermark>;
+}
+//# sourceMappingURL=replica-poller.d.ts.map

package/out/zero-cache/src/services/change-streamer/replica-poller.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replica-poller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/replica-poller.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAI3D;;;;GAIG;AACH,qBAAa,aAAa;;IACxB,QAAQ,CAAC,EAAE,oBAAoB;gBAQnB,EAAE,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM;IAQ/C,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC;CAyBnC"}