@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.40

package/out/zero-cache/src/services/change-streamer/forwarder.js.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#backupLagEventCount","#lastBackupLagWarnMs","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusTimeoutProportion: number;\n  flowControlSlowSubscriberGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #earlyReleaseOptions: EarlyReleaseOptions | undefined;\n  readonly #flowControlSlowSubscriberGracePeriodMs: number | undefined;\n\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  // The backup-replicator is never disconnected as a laggard, so its lag\n  // detections are logged (rather than acted upon). Track the running total and\n  // throttle the warning to at most once per grace period to avoid spamming the\n  // log for the duration of a slow catchup.\n  #backupLagEventCount = 0;\n  #lastBackupLagWarnMs: number | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {\n      flowControlConsensusTimeoutProportion: 2.0,\n    },\n  ) {\n    this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    this.#earlyReleaseOptions =\n      opts.flowControlConsensusTimeoutProportion < 0\n        ? undefined\n        : {\n            consensusTimeoutProportion:\n              opts.flowControlConsensusTimeoutProportion,\n            setTimeoutFn: this.#setTimeout,\n            clearTimeoutFn: this.#clearTimeout,\n          };\n    this.#flowControlSlowSubscriberGracePeriodMs =\n      opts.flowControlSlowSubscriberGracePeriodMs;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(\n      () => this.checkSubscriberProgress(performance.now()),\n      1000,\n    );\n  }\n\n  /**\n   * Runs one progress-monitor tick: samples each active subscriber's change\n   * rate, logs slow broadcasts, and disconnects subscribers that have been\n   * continuously lagging for longer than the configured grace period.\n   *\n   * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n   * than private) so tests can drive a single tick with an explicit `now`.\n   */\n  checkSubscriberProgress(now: number) {\n    let timedOut: Map<Subscriber, SubscriberStats> | undefined;\n    let slowestOnTime: SubscriberStats | undefined;\n\n    for (const sub of this.#active) {\n      const stats = sub.sampleProcessRate(now).getStats();\n      if (stats.missedLastTimeout) {\n        (timedOut ??= new Map()).set(sub, stats);\n      } else if (\n        stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n      ) {\n        slowestOnTime = stats;\n      }\n    }\n    this.#currentBroadcast?.logSlowRequests(now);\n\n    if (\n      this.#flowControlSlowSubscriberGracePeriodMs &&\n      timedOut?.size &&\n      slowestOnTime\n    ) {\n      for (const [sub, stats] of timedOut.entries()) {\n        const rate =\n          stats.processRate < slowestOnTime.processRate\n            ? 'lagging'\n            : 'catching-up';\n        const laggingDuration = sub.reportChangeRate(now, rate);\n        if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n          if (sub.mode === 'backup') {\n            // The backup-replicator can only be detected as a laggard during\n            // initial catchup; after that it is required for consensus. If it\n            // triggers laggard detection in the former case, just log and\n            // continue; as disconnecting it would otherwise kill the\n            // change-streamer task altogether.\n            this.#backupLagEventCount++;\n            const grace = this.#flowControlSlowSubscriberGracePeriodMs;\n            if (\n              this.#lastBackupLagWarnMs === undefined ||\n              now - this.#lastBackupLagWarnMs >= grace\n            ) {\n              this.#lastBackupLagWarnMs = now;\n              this.#lc.warn?.(\n                `backup subscriber ${sub.id} has lagged for ${laggingDuration}ms ` +\n                  `(${this.#backupLagEventCount} lag events so far). ` +\n                  `this is only expected during initial catchup`,\n                {stats},\n              );\n            }\n          } else {\n            this.#lc.warn?.(\n              `serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n              {stats},\n            );\n            sub.close(\n              StreamTooFarBehind,\n              `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n            );\n          }\n        }\n      }\n    }\n  }\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const start = performance.now();\n    const broadcast = new Broadcast(\n      this.#lc,\n      this.#active.values(),\n      entry,\n      this.#earlyReleaseOptions,\n    );\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAoBA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB;CACA;CAMA,uBAAuB;CACvB;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAC1B,IAAI,IAAI,SAAS,UAAU;IAMzB,KAAKM;IACL,MAAM,QAAQ,KAAKN;IACnB,IACE,KAAKO,yBAAyB,KAAA,KAC9B,MAAM,KAAKA,wBAAwB,OACnC;KACA,KAAKA,uBAAuB;KAC5B,KAAKb,IAAI,OACP,qBAAqB,IAAI,GAAG,kBAAkB,gBAAgB,MACxD,KAAKY,qBAAqB,oEAEhC,EAAC,MAAK,CACR;IACF;GACF,OAAO;IACL,KAAKZ,IAAI,OACP,sBAAsB,IAAI,GAAG,kBAAkB,gBAAgB,oBAC/D,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EAEJ;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKI,gBACP,KAAKV,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKY,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKZ,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKH,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKU,yBAAyB,MAAM,EAAE;EAGtC,KAAKJ,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKR,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKU,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKV,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}
+{"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#backupLagEventCount","#lastBackupLagWarnMs","#flushPendingWithoutTracking","#inTransaction","#pending","#updateActiveSubscribers","#flushScheduled","#flushImmediateId","#drainPending"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusTimeoutProportion: number;\n  flowControlSlowSubscriberGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nconst FORWARD_BATCH_SIZE = 64;\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #earlyReleaseOptions: EarlyReleaseOptions | undefined;\n  readonly #flowControlSlowSubscriberGracePeriodMs: number | undefined;\n\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #pending: WatermarkedChange[] = [];\n  #flushScheduled = false;\n  #flushImmediateId: NodeJS.Immediate | undefined = undefined;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  // The backup-replicator is never disconnected as a laggard, so its lag\n  // detections are logged (rather than acted upon). Track the running total and\n  // throttle the warning to at most once per grace period to avoid spamming the\n  // log for the duration of a slow catchup.\n  #backupLagEventCount = 0;\n  #lastBackupLagWarnMs: number | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {\n      flowControlConsensusTimeoutProportion: 2.0,\n    },\n  ) {\n    this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    this.#earlyReleaseOptions =\n      opts.flowControlConsensusTimeoutProportion < 0\n        ? undefined\n        : {\n            consensusTimeoutProportion:\n              opts.flowControlConsensusTimeoutProportion,\n            setTimeoutFn: this.#setTimeout,\n            clearTimeoutFn: this.#clearTimeout,\n          };\n    this.#flowControlSlowSubscriberGracePeriodMs =\n      opts.flowControlSlowSubscriberGracePeriodMs;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(\n      () => this.checkSubscriberProgress(performance.now()),\n      1000,\n    );\n  }\n\n  /**\n   * Runs one progress-monitor tick: samples each active subscriber's change\n   * rate, logs slow broadcasts, and disconnects subscribers that have been\n   * continuously lagging for longer than the configured grace period.\n   *\n   * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n   * than private) so tests can drive a single tick with an explicit `now`.\n   */\n  checkSubscriberProgress(now: number) {\n    let timedOut: Map<Subscriber, SubscriberStats> | undefined;\n    let slowestOnTime: SubscriberStats | undefined;\n\n    for (const sub of this.#active) {\n      const stats = sub.sampleProcessRate(now).getStats();\n      if (stats.missedLastTimeout) {\n        (timedOut ??= new Map()).set(sub, stats);\n      } else if (\n        stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n      ) {\n        slowestOnTime = stats;\n      }\n    }\n    this.#currentBroadcast?.logSlowRequests(now);\n\n    if (\n      this.#flowControlSlowSubscriberGracePeriodMs &&\n      timedOut?.size &&\n      slowestOnTime\n    ) {\n      for (const [sub, stats] of timedOut.entries()) {\n        const rate =\n          stats.processRate < slowestOnTime.processRate\n            ? 'lagging'\n            : 'catching-up';\n        const laggingDuration = sub.reportChangeRate(now, rate);\n        if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n          if (sub.mode === 'backup') {\n            // The backup-replicator can only be detected as a laggard during\n            // initial catchup; after that it is required for consensus. If it\n            // triggers laggard detection in the former case, just log and\n            // continue; as disconnecting it would otherwise kill the\n            // change-streamer task altogether.\n            this.#backupLagEventCount++;\n            const grace = this.#flowControlSlowSubscriberGracePeriodMs;\n            if (\n              this.#lastBackupLagWarnMs === undefined ||\n              now - this.#lastBackupLagWarnMs >= grace\n            ) {\n              this.#lastBackupLagWarnMs = now;\n              this.#lc.warn?.(\n                `backup subscriber ${sub.id} has lagged for ${laggingDuration}ms ` +\n                  `(${this.#backupLagEventCount} lag events so far). ` +\n                  `this is only expected during initial catchup`,\n                {stats},\n              );\n            }\n          } else {\n            this.#lc.warn?.(\n              `serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n              {stats},\n            );\n            sub.close(\n              StreamTooFarBehind,\n              `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n            );\n          }\n        }\n      }\n    }\n  }\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#flushPendingWithoutTracking();\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    this.#flushPendingWithoutTracking();\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#flushPendingWithoutTracking();\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward buffers changes across the current event loop turn\n   * and dispatches them in batches. The change-streamer should call and await\n   * {@link forwardWithFlowControl()} occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    this.#pending.push(entry);\n\n    if (\n      this.#queued.size > 0 &&\n      (entry[1] === 'commit' || entry[1] === 'rollback')\n    ) {\n      this.#flushPendingWithoutTracking();\n    }\n    this.#updateActiveSubscribers(entry[1]);\n\n    if (this.#pending.length >= FORWARD_BATCH_SIZE) {\n      this.#flushPendingWithoutTracking();\n    } else if (!this.#flushScheduled && this.#pending.length > 0) {\n      this.#flushScheduled = true;\n      this.#flushImmediateId = setImmediate(() =>\n        this.#flushPendingWithoutTracking(),\n      );\n    }\n  }\n\n  sendStatus(status: Status) {\n    this.#flushPendingWithoutTracking();\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    this.#pending.push(entry);\n\n    const batch = this.#drainPending();\n    const start = performance.now();\n    const broadcast = new Broadcast(\n      this.#lc,\n      this.#active.values(),\n      batch,\n      this.#earlyReleaseOptions,\n    );\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #drainPending(): WatermarkedChange[] {\n    if (this.#flushImmediateId !== undefined) {\n      clearImmediate(this.#flushImmediateId);\n      this.#flushImmediateId = undefined;\n    }\n    this.#flushScheduled = false;\n    if (this.#pending.length === 0) {\n      return [];\n    }\n    const batch = this.#pending;\n    this.#pending = [];\n    return batch;\n  }\n\n  #flushPendingWithoutTracking() {\n    const batch = this.#drainPending();\n    if (batch.length > 0) {\n      Broadcast.withoutTracking(this.#active.values(), batch);\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAoBA,IAAM,qBAAqB;AAE3B,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB,WAAgC,CAAC;CACjC,kBAAkB;CAClB,oBAAkD,KAAA;CAElD;CACA;CAMA,uBAAuB;CACvB;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAC1B,IAAI,IAAI,SAAS,UAAU;IAMzB,KAAKM;IACL,MAAM,QAAQ,KAAKN;IACnB,IACE,KAAKO,yBAAyB,KAAA,KAC9B,MAAM,KAAKA,wBAAwB,OACnC;KACA,KAAKA,uBAAuB;KAC5B,KAAKb,IAAI,OACP,qBAAqB,IAAI,GAAG,kBAAkB,gBAAgB,MACxD,KAAKY,qBAAqB,oEAEhC,EAAC,MAAK,CACR;IACF;GACF,OAAO;IACL,KAAKZ,IAAI,OACP,sBAAsB,IAAI,GAAG,kBAAkB,gBAAgB,oBAC/D,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EAEJ;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;EACnC,KAAKI,6BAA6B;CACpC;;;;;;CAOA,IAAI,KAAiB;EACnB,KAAKA,6BAA6B;EAClC,IAAI,KAAKC,gBACP,KAAKX,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKW,6BAA6B;EAClC,KAAKX,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,KAAKY,SAAS,KAAK,KAAK;EAExB,IACE,KAAKZ,QAAQ,OAAO,MACnB,MAAM,OAAO,YAAY,MAAM,OAAO,aAEvC,KAAKU,6BAA6B;EAEpC,KAAKG,yBAAyB,MAAM,EAAE;EAEtC,IAAI,KAAKD,SAAS,UAAU,oBAC1B,KAAKF,6BAA6B;OAC7B,IAAI,CAAC,KAAKI,mBAAmB,KAAKF,SAAS,SAAS,GAAG;GAC5D,KAAKE,kBAAkB;GACvB,KAAKC,oBAAoB,mBACvB,KAAKL,6BAA6B,CACpC;EACF;CACF;CAEA,WAAW,QAAgB;EACzB,KAAKA,6BAA6B;EAClC,KAAK,MAAM,OAAO,KAAKX,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,KAAKa,SAAS,KAAK,KAAK;EAExB,MAAM,QAAQ,KAAKI,cAAc;EACjC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKpB,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKY,yBAAyB,MAAM,EAAE;EAGtC,KAAKN,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,gBAAqC;EACnC,IAAI,KAAKQ,sBAAsB,KAAA,GAAW;GACxC,eAAe,KAAKA,iBAAiB;GACrC,KAAKA,oBAAoB,KAAA;EAC3B;EACA,KAAKD,kBAAkB;EACvB,IAAI,KAAKF,SAAS,WAAW,GAC3B,OAAO,CAAC;EAEV,MAAM,QAAQ,KAAKA;EACnB,KAAKA,WAAW,CAAC;EACjB,OAAO;CACT;CAEA,+BAA+B;EAC7B,MAAM,QAAQ,KAAKI,cAAc;EACjC,IAAI,MAAM,SAAS,GACjB,UAAU,gBAAgB,KAAKjB,QAAQ,OAAO,GAAG,KAAK;CAE1D;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKA,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKW,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKX,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.d.ts

@@ -0,0 +1,53 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Source } from '../../types/streams.ts';
+import { LitestreamController } from '../litestream/litestream-controller.ts';
+import type { BackedUpWatermark } from './backup-monitor.ts';
+/** The subset of {@link LitestreamController} used by the requester. */
+export type SyncController = Pick<LitestreamController, 'sync' | 'close'>;
+export type SyncRequesterConfig = {
+    /**
+     * Interval between sync requests. This paces how often litestream seals
+     * (and uploads) a new LTX file.
+     */
+    intervalMs: number;
+};
+/**
+ * The LitestreamSyncRequester drives litestream v5 backups and tracks which
+ * replica watermark is durably backed up, emitting it as a stream of
+ * {@link BackedUpWatermark}s.
+ *
+ * It periodically reads the replica's current watermark `W` and then asks
+ * litestream to sync and wait for the upload (`POST /sync {wait: true}`).
+ * litestream satisfies the request with a sync pass that *starts after* the
+ * request, and a pass syncs up to the end of the WAL, so a successful response
+ * means that every transaction committed before the request, including the
+ * one that wrote `W`, has been uploaded to the backup. No part of the backup
+ * needs to be read to determine this.
+ *
+ * Requests continue even when `W` is already backed up, so that changes that
+ * don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)
+ * are also backed up at this cadence rather than only at litestream's slower
+ * backstop interval. A request on an idle replica is a local no-op for
+ * litestream: there is nothing to seal or upload, and no backup reads.
+ *
+ * At most one request is outstanding at a time. Failed or timed-out requests
+ * are retried with a backoff that starts short and, once it grows to the
+ * interval, defers to the regular ticks. Nothing is published until a request
+ * succeeds.
+ */
+export declare class LitestreamSyncRequester {
+    #private;
+    constructor(lc: LogContext, replicaFile: string, config: SyncRequesterConfig, litestream?: SyncController);
+    /**
+     * Starts requesting syncs and pushes backed-up watermarks to the returned
+     * stream. Must only be called once.
+     */
+    start(): Source<BackedUpWatermark>;
+    /**
+     * Requests a sync unless a request is already outstanding. Resolves when
+     * this tick's request (or the outstanding one) completes.
+     * Exported for testing.
+     */
+    readonly tick: () => Promise<void>;
+}
+//# sourceMappingURL=litestream-sync-requester.d.ts.map

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream-sync-requester.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/litestream-sync-requester.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAEnD,OAAO,EAAC,oBAAoB,EAAC,MAAM,wCAAwC,CAAC;AAE5E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAoC3D,wEAAwE;AACxE,MAAM,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC;AAE1E,MAAM,MAAM,mBAAmB,GAAG;IAChC;;;OAGG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,qBAAa,uBAAuB;;IAelC,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,mBAAmB,EAC3B,UAAU,GAAE,cAA0D,EAmBvE;IAED;;;OAGG;IACH,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC,CAQjC;IAED;;;;OAIG;IACH,QAAQ,CAAC,IAAI,QAAO,OAAO,CAAC,IAAI,CAAC,CAe/B;CAqDH"}

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.js

@@ -0,0 +1,156 @@
+import { assert } from "../../../../shared/src/asserts.js";
+import { parse, valita_exports } from "../../../../shared/src/valita.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { RunningState } from "../running-state.js";
+import { Subscription } from "../../types/subscription.js";
+import { LitestreamController } from "../litestream/litestream-controller.js";
+//#region ../zero-cache/src/services/change-streamer/litestream-sync-requester.ts
+/**
+* How long litestream waits for a requested sync (and its upload) before
+* ending the request. A request that times out is simply retried on a later
+* tick: litestream only abandons the *wait*, never the sync or upload itself,
+* so this bounds request lifetime without affecting backup progress.
+*/
+var SERVER_TIMEOUT_SECONDS = 60;
+/**
+* Client-side timeout, slightly above the server timeout so that litestream,
+* rather than the client, ends a slow request.
+*/
+var CLIENT_TIMEOUT_MS = (SERVER_TIMEOUT_SECONDS + 5) * 1e3;
+var ERROR_LOG_INTERVAL_MS = 6e4;
+/**
+* Initial delay before retrying a failed request, doubled on each consecutive
+* failure until it reaches the configured interval, after which the regular
+* ticks alone retry, so that a busy litestream sees no more requests than the
+* interval allows. The first request is typically issued before litestream's
+* control socket is listening; retrying quickly (rather than after a full
+* interval) keeps the initial backup confirmation, which gates readiness, from
+* waiting out an interval.
+*/
+var MIN_RETRY_DELAY_MS = 250;
+var localWatermarkSchema = valita_exports.object({
+	stateVersion: valita_exports.string(),
+	writeTimeMs: valita_exports.number()
+});
+/**
+* The LitestreamSyncRequester drives litestream v5 backups and tracks which
+* replica watermark is durably backed up, emitting it as a stream of
+* {@link BackedUpWatermark}s.
+*
+* It periodically reads the replica's current watermark `W` and then asks
+* litestream to sync and wait for the upload (`POST /sync {wait: true}`).
+* litestream satisfies the request with a sync pass that *starts after* the
+* request, and a pass syncs up to the end of the WAL, so a successful response
+* means that every transaction committed before the request, including the
+* one that wrote `W`, has been uploaded to the backup. No part of the backup
+* needs to be read to determine this.
+*
+* Requests continue even when `W` is already backed up, so that changes that
+* don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)
+* are also backed up at this cadence rather than only at litestream's slower
+* backstop interval. A request on an idle replica is a local no-op for
+* litestream: there is nothing to seal or upload, and no backup reads.
+*
+* At most one request is outstanding at a time. Failed or timed-out requests
+* are retried with a backoff that starts short and, once it grows to the
+* interval, defers to the regular ticks. Nothing is published until a request
+* succeeds.
+*/
+var LitestreamSyncRequester = class {
+	#lc;
+	#state = new RunningState("litestream-sync-requester");
+	#config;
+	#litestream;
+	#readLocalWatermark;
+	#stream;
+	#timer;
+	#retryTimer;
+	#retryDelayMs = MIN_RETRY_DELAY_MS;
+	#inFlight;
+	#backedUp;
+	#lastErrorLogMs = 0;
+	constructor(lc, replicaFile, config, litestream = new LitestreamController(lc, replicaFile)) {
+		this.#lc = lc.withContext("component", "litestream-sync-requester");
+		this.#config = config;
+		this.#litestream = litestream;
+		const db = new Database(this.#lc, replicaFile, { readonly: true });
+		this.#readLocalWatermark = db.prepare(`SELECT stateVersion, writeTimeMs FROM "_zero.replicationState"`);
+		this.#stream = Subscription.create({ cleanup: () => {
+			clearInterval(this.#timer);
+			clearTimeout(this.#retryTimer);
+			this.#state.stop(this.#lc);
+			this.#litestream.close();
+			db.close();
+		} });
+	}
+	/**
+	* Starts requesting syncs and pushes backed-up watermarks to the returned
+	* stream. Must only be called once.
+	*/
+	start() {
+		assert(this.#timer === void 0, `Already called start()`);
+		this.#lc.info?.(`requesting litestream syncs every ${this.#config.intervalMs} ms`);
+		this.#timer = setInterval(this.tick, this.#config.intervalMs);
+		this.tick();
+		return this.#stream;
+	}
+	/**
+	* Requests a sync unless a request is already outstanding. Resolves when
+	* this tick's request (or the outstanding one) completes.
+	* Exported for testing.
+	*/
+	tick = () => {
+		if (!this.#state.shouldRun() || this.#inFlight) return this.#inFlight ?? Promise.resolve();
+		let local;
+		try {
+			local = parse(this.#readLocalWatermark.get(), localWatermarkSchema);
+		} catch (e) {
+			this.#lc.warn?.(`unable to read local watermark`, e);
+			return Promise.resolve();
+		}
+		this.#inFlight = this.#requestSync(local).finally(() => {
+			this.#inFlight = void 0;
+		});
+		return this.#inFlight;
+	};
+	async #requestSync(local) {
+		try {
+			const result = await this.#litestream.sync({
+				wait: true,
+				timeoutMs: CLIENT_TIMEOUT_MS,
+				serverTimeoutSeconds: SERVER_TIMEOUT_SECONDS
+			}, this.#state.signal);
+			this.#retryDelayMs = MIN_RETRY_DELAY_MS;
+			this.#publish(local, Date.now());
+			this.#lc.debug?.(`backed up watermark ${local.stateVersion}`, result);
+		} catch (e) {
+			if (!this.#state.shouldRun()) return;
+			this.#scheduleRetry();
+			const now = Date.now();
+			if (now - this.#lastErrorLogMs >= ERROR_LOG_INTERVAL_MS) {
+				this.#lastErrorLogMs = now;
+				this.#lc.info?.(`backup of watermark ${local.stateVersion} not yet confirmed. retrying`, e);
+			}
+		}
+	}
+	#scheduleRetry() {
+		clearTimeout(this.#retryTimer);
+		const delayMs = this.#retryDelayMs;
+		if (delayMs >= this.#config.intervalMs) return;
+		this.#retryDelayMs = delayMs * 2;
+		this.#retryTimer = setTimeout(this.tick, delayMs);
+	}
+	#publish({ stateVersion, writeTimeMs }, backupTimeMs) {
+		if (this.#backedUp !== void 0 && stateVersion <= this.#backedUp) return;
+		this.#backedUp = stateVersion;
+		this.#stream.push({
+			watermark: stateVersion,
+			writeTimeMs,
+			backupTimeMs
+		});
+	}
+};
+//#endregion
+export { LitestreamSyncRequester };
+
+//# sourceMappingURL=litestream-sync-requester.js.map

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream-sync-requester.js","names":["#lc","#state","#config","#litestream","#readLocalWatermark","#stream","#timer","#retryTimer","#inFlight","#requestSync","#retryDelayMs","#publish","#scheduleRetry","#lastErrorLogMs","#backedUp"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream-sync-requester.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database, type Statement} from '../../../../zqlite/src/db.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {LitestreamController} from '../litestream/litestream-controller.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\n/**\n * How long litestream waits for a requested sync (and its upload) before\n * ending the request. A request that times out is simply retried on a later\n * tick: litestream only abandons the *wait*, never the sync or upload itself,\n * so this bounds request lifetime without affecting backup progress.\n */\nconst SERVER_TIMEOUT_SECONDS = 60;\n\n/**\n * Client-side timeout, slightly above the server timeout so that litestream,\n * rather than the client, ends a slow request.\n */\nconst CLIENT_TIMEOUT_MS = (SERVER_TIMEOUT_SECONDS + 5) * 1000;\n\nconst ERROR_LOG_INTERVAL_MS = 60_000;\n\n/**\n * Initial delay before retrying a failed request, doubled on each consecutive\n * failure until it reaches the configured interval, after which the regular\n * ticks alone retry, so that a busy litestream sees no more requests than the\n * interval allows. The first request is typically issued before litestream's\n * control socket is listening; retrying quickly (rather than after a full\n * interval) keeps the initial backup confirmation, which gates readiness, from\n * waiting out an interval.\n */\nconst MIN_RETRY_DELAY_MS = 250;\n\nconst localWatermarkSchema = v.object({\n  stateVersion: v.string(),\n  writeTimeMs: v.number(),\n});\n\ntype LocalWatermark = v.Infer<typeof localWatermarkSchema>;\n\n/** The subset of {@link LitestreamController} used by the requester. */\nexport type SyncController = Pick<LitestreamController, 'sync' | 'close'>;\n\nexport type SyncRequesterConfig = {\n  /**\n   * Interval between sync requests. This paces how often litestream seals\n   * (and uploads) a new LTX file.\n   */\n  intervalMs: number;\n};\n\n/**\n * The LitestreamSyncRequester drives litestream v5 backups and tracks which\n * replica watermark is durably backed up, emitting it as a stream of\n * {@link BackedUpWatermark}s.\n *\n * It periodically reads the replica's current watermark `W` and then asks\n * litestream to sync and wait for the upload (`POST /sync {wait: true}`).\n * litestream satisfies the request with a sync pass that *starts after* the\n * request, and a pass syncs up to the end of the WAL, so a successful response\n * means that every transaction committed before the request, including the\n * one that wrote `W`, has been uploaded to the backup. No part of the backup\n * needs to be read to determine this.\n *\n * Requests continue even when `W` is already backed up, so that changes that\n * don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)\n * are also backed up at this cadence rather than only at litestream's slower\n * backstop interval. A request on an idle replica is a local no-op for\n * litestream: there is nothing to seal or upload, and no backup reads.\n *\n * At most one request is outstanding at a time. Failed or timed-out requests\n * are retried with a backoff that starts short and, once it grows to the\n * interval, defers to the regular ticks. Nothing is published until a request\n * succeeds.\n */\nexport class LitestreamSyncRequester {\n  readonly #lc: LogContext;\n  readonly #state = new RunningState('litestream-sync-requester');\n  readonly #config: SyncRequesterConfig;\n  readonly #litestream: SyncController;\n  readonly #readLocalWatermark: Statement;\n  readonly #stream: Subscription<BackedUpWatermark>;\n\n  #timer: NodeJS.Timeout | undefined;\n  #retryTimer: NodeJS.Timeout | undefined;\n  #retryDelayMs = MIN_RETRY_DELAY_MS;\n  #inFlight: Promise<void> | undefined;\n  #backedUp: string | undefined;\n  #lastErrorLogMs = 0;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    config: SyncRequesterConfig,\n    litestream: SyncController = new LitestreamController(lc, replicaFile),\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-sync-requester');\n    this.#config = config;\n    this.#litestream = litestream;\n\n    const db = new Database(this.#lc, replicaFile, {readonly: true});\n    this.#readLocalWatermark = db.prepare(\n      `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n    );\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        clearInterval(this.#timer);\n        clearTimeout(this.#retryTimer);\n        this.#state.stop(this.#lc);\n        this.#litestream.close();\n        db.close();\n      },\n    });\n  }\n\n  /**\n   * Starts requesting syncs and pushes backed-up watermarks to the returned\n   * stream. Must only be called once.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#timer === undefined, `Already called start()`);\n    this.#lc.info?.(\n      `requesting litestream syncs every ${this.#config.intervalMs} ms`,\n    );\n    this.#timer = setInterval(this.tick, this.#config.intervalMs);\n    void this.tick();\n    return this.#stream;\n  }\n\n  /**\n   * Requests a sync unless a request is already outstanding. Resolves when\n   * this tick's request (or the outstanding one) completes.\n   * Exported for testing.\n   */\n  readonly tick = (): Promise<void> => {\n    if (!this.#state.shouldRun() || this.#inFlight) {\n      return this.#inFlight ?? Promise.resolve();\n    }\n    let local: LocalWatermark;\n    try {\n      local = v.parse(this.#readLocalWatermark.get(), localWatermarkSchema);\n    } catch (e) {\n      this.#lc.warn?.(`unable to read local watermark`, e);\n      return Promise.resolve();\n    }\n    this.#inFlight = this.#requestSync(local).finally(() => {\n      this.#inFlight = undefined;\n    });\n    return this.#inFlight;\n  };\n\n  async #requestSync(local: LocalWatermark) {\n    // `local` must be read before the request is issued: litestream only\n    // guarantees that transactions committed before the request are synced.\n    try {\n      const result = await this.#litestream.sync(\n        {\n          wait: true,\n          timeoutMs: CLIENT_TIMEOUT_MS,\n          serverTimeoutSeconds: SERVER_TIMEOUT_SECONDS,\n        },\n        this.#state.signal,\n      );\n      this.#retryDelayMs = MIN_RETRY_DELAY_MS;\n      this.#publish(local, Date.now());\n      this.#lc.debug?.(`backed up watermark ${local.stateVersion}`, result);\n    } catch (e) {\n      if (!this.#state.shouldRun()) {\n        return; // shutting down\n      }\n      // Expected while litestream is starting up or busy with a long sync or\n      // upload (e.g. the initial snapshot of a large replica); the request is\n      // retried after a backoff, or on the next tick if that comes first.\n      this.#scheduleRetry();\n      const now = Date.now();\n      if (now - this.#lastErrorLogMs >= ERROR_LOG_INTERVAL_MS) {\n        this.#lastErrorLogMs = now;\n        this.#lc.info?.(\n          `backup of watermark ${local.stateVersion} not yet confirmed. retrying`,\n          e,\n        );\n      }\n    }\n  }\n\n  #scheduleRetry() {\n    clearTimeout(this.#retryTimer);\n    const delayMs = this.#retryDelayMs;\n    if (delayMs >= this.#config.intervalMs) {\n      return; // backed off to the interval; the regular ticks retry\n    }\n    this.#retryDelayMs = delayMs * 2;\n    this.#retryTimer = setTimeout(this.tick, delayMs);\n  }\n\n  #publish({stateVersion, writeTimeMs}: LocalWatermark, backupTimeMs: number) {\n    if (this.#backedUp !== undefined && stateVersion <= this.#backedUp) {\n      return;\n    }\n    this.#backedUp = stateVersion;\n    this.#stream.push({watermark: stateVersion, writeTimeMs, backupTimeMs});\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AAgBA,IAAM,yBAAyB;;;;;AAM/B,IAAM,qBAAqB,yBAAyB,KAAK;AAEzD,IAAM,wBAAwB;;;;;;;;;;AAW9B,IAAM,qBAAqB;AAE3B,IAAM,uBAAuB,eAAE,OAAO;CACpC,cAAc,eAAE,OAAO;CACvB,aAAa,eAAE,OAAO;AACxB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;AAuCD,IAAa,0BAAb,MAAqC;CACnC;CACA,SAAkB,IAAI,aAAa,2BAA2B;CAC9D;CACA;CACA;CACA;CAEA;CACA;CACA,gBAAgB;CAChB;CACA;CACA,kBAAkB;CAElB,YACE,IACA,aACA,QACA,aAA6B,IAAI,qBAAqB,IAAI,WAAW,GACrE;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAKE,UAAU;EACf,KAAKC,cAAc;EAEnB,MAAM,KAAK,IAAI,SAAS,KAAKH,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,KAAKI,sBAAsB,GAAG,QAC5B,gEACF;EACA,KAAKC,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAKC,MAAM;GACzB,aAAa,KAAKC,WAAW;GAC7B,KAAKN,OAAO,KAAK,KAAKD,GAAG;GACzB,KAAKG,YAAY,MAAM;GACvB,GAAG,MAAM;EACX,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKG,WAAW,KAAA,GAAW,wBAAwB;EAC1D,KAAKN,IAAI,OACP,qCAAqC,KAAKE,QAAQ,WAAW,IAC/D;EACA,KAAKI,SAAS,YAAY,KAAK,MAAM,KAAKJ,QAAQ,UAAU;EAC5D,KAAU,KAAK;EACf,OAAO,KAAKG;CACd;;;;;;CAOA,aAAqC;EACnC,IAAI,CAAC,KAAKJ,OAAO,UAAU,KAAK,KAAKO,WACnC,OAAO,KAAKA,aAAa,QAAQ,QAAQ;EAE3C,IAAI;EACJ,IAAI;GACF,QAAQ,MAAQ,KAAKJ,oBAAoB,IAAI,GAAG,oBAAoB;EACtE,SAAS,GAAG;GACV,KAAKJ,IAAI,OAAO,kCAAkC,CAAC;GACnD,OAAO,QAAQ,QAAQ;EACzB;EACA,KAAKQ,YAAY,KAAKC,aAAa,KAAK,EAAE,cAAc;GACtD,KAAKD,YAAY,KAAA;EACnB,CAAC;EACD,OAAO,KAAKA;CACd;CAEA,MAAMC,aAAa,OAAuB;EAGxC,IAAI;GACF,MAAM,SAAS,MAAM,KAAKN,YAAY,KACpC;IACE,MAAM;IACN,WAAW;IACX,sBAAsB;GACxB,GACA,KAAKF,OAAO,MACd;GACA,KAAKS,gBAAgB;GACrB,KAAKC,SAAS,OAAO,KAAK,IAAI,CAAC;GAC/B,KAAKX,IAAI,QAAQ,uBAAuB,MAAM,gBAAgB,MAAM;EACtE,SAAS,GAAG;GACV,IAAI,CAAC,KAAKC,OAAO,UAAU,GACzB;GAKF,KAAKW,eAAe;GACpB,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,MAAM,KAAKC,mBAAmB,uBAAuB;IACvD,KAAKA,kBAAkB;IACvB,KAAKb,IAAI,OACP,uBAAuB,MAAM,aAAa,+BAC1C,CACF;GACF;EACF;CACF;CAEA,iBAAiB;EACf,aAAa,KAAKO,WAAW;EAC7B,MAAM,UAAU,KAAKG;EACrB,IAAI,WAAW,KAAKR,QAAQ,YAC1B;EAEF,KAAKQ,gBAAgB,UAAU;EAC/B,KAAKH,cAAc,WAAW,KAAK,MAAM,OAAO;CAClD;CAEA,SAAS,EAAC,cAAc,eAA8B,cAAsB;EAC1E,IAAI,KAAKO,cAAc,KAAA,KAAa,gBAAgB,KAAKA,WACvD;EAEF,KAAKA,YAAY;EACjB,KAAKT,QAAQ,KAAK;GAAC,WAAW;GAAc;GAAa;EAAY,CAAC;CACxE;AACF"}

package/out/zero-cache/src/services/change-streamer/restore-admission.d.ts

@@ -0,0 +1,243 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { LocalReplica } from './subscribe.ts';
+/**
+ * The default time to wait for a subscriber that was granted permission to
+ * restore to reconnect (i.e. to open a replacement reservation) after its
+ * held connection closes, or after its replacement reservation closes
+ * without subscribing (e.g. to retry a failed restore). This covers the
+ * subscriber's process exit and restart (e.g. a container restart, the
+ * backoff of which starts at 10 seconds in Kubernetes), but not the restore
+ * itself, which is tracked by the liveness of its snapshot reservation.
+ */
+export declare const DEFAULT_PENDING_GRANT_TTL_MS = 90000;
+/**
+ * The default time for which a replacement continues to be counted after its
+ * subscriber has caught up, which covers the time for the subscriber to start
+ * serving (i.e. become ready) after catching up. See
+ * {@link RestoreAdmissionOptions.readinessGracePeriodMs}.
+ */
+export declare const DEFAULT_READINESS_GRACE_PERIOD_MS = 5000;
+export type RestoreAdmissionOptions = {
+    /** The replica version of the replication-manager. */
+    replicaVersion: string;
+    /**
+     * The maximum number of subscribers that may be restoring (or granted
+     * permission to restore) at the same time.
+     */
+    maxConcurrentRestores: number;
+    pendingGrantTTLMs?: number | undefined;
+    /**
+     * The time for which a replacement continues to be counted after its
+     * subscriber has caught up, so that the next subscriber is not granted
+     * permission to restore (and stop serving) before the restored one has
+     * started serving. Catching up is followed by the subscriber's readiness
+     * checks (e.g. waiting for the next replication lag report), and by the
+     * container runner's detection of its readiness.
+     */
+    readinessGracePeriodMs?: number | undefined;
+    setTimeoutFn?: typeof setTimeout | undefined;
+    clearTimeoutFn?: typeof clearTimeout | undefined;
+};
+export type AdmissionRequest = {
+    /**
+     * Called when the request is queued, and again whenever its (1-based)
+     * position in the queue changes.
+     */
+    onQueued: (position: number) => void;
+    /** Called (once) when the request is granted. */
+    onGranted: () => void;
+};
+/** A subscribed replacement, returned by {@link RestoreAdmission.reservationSubscribed}. */
+export type Replacement = {
+    /**
+     * Called when the subscriber has caught up. The replacement is released
+     * after the readiness grace period.
+     */
+    caughtUp: () => void;
+    /** Called when the subscription closes. The replacement is released. */
+    closed: () => void;
+};
+export type AdmissionTicket = {
+    /**
+     * Must be called when the requesting connection closes. A queued request
+     * is removed from the queue. For a granted request, this starts the
+     * timer within which the subscriber is expected to reconnect and open a
+     * replacement reservation.
+     */
+    release(): void;
+};
+/**
+ * Limits the number of view-syncers that replace their replica at the same
+ * time, so that view-syncers switching over from an incompatible
+ * replication-manager (e.g. after a resync) do not all stop serving at once.
+ *
+ * Capacity is consumed by:
+ * * Replacement restores: snapshot reservations of subscribers that are
+ *   replacing an (incompatible or outdated) local replica, i.e. subscribers
+ *   that were serving. Restores onto an empty disk (e.g. a new pod) do not
+ *   take a serving subscriber out of service, and are not counted. The
+ *   reservation's connection is held open for the duration of the restore,
+ *   and the subscription that follows it is held open until the restored
+ *   subscriber has caught up, so their liveness tracks that of the
+ *   replacement without a timeout.
+ * * Outstanding grants: permission to restore that has not yet been
+ *   followed by a replacement reservation (e.g. while the subscriber drains
+ *   and restarts). Grants are not matched to reservations by identity:
+ *   any replacement reservation consumes an outstanding grant, and a grant
+ *   that is not consumed expires a short time after the granted connection
+ *   closes (e.g. if the subscriber reconnected to a different
+ *   replication-manager or was deleted).
+ *
+ * ## Life cycle of a switchover restore
+ *
+ * The following traces a view-syncer (VS) with a replica at version `R1`,
+ * served by an old replication-manager (RM) at `R1`, while a new RM (e.g.
+ * after a resync) is at `R2 > R1`. The RestoreAdmission is that of the new
+ * RM. Terms in `code` are those used in the code.
+ *
+ * 1. **Drain.** The old RM receives a SIGTERM, and its `DrainTracker` sends
+ *    `status {draining: true}` to its `drainable` (i.e. serving) subscribers.
+ *    It continues to stream changes to them, and exits once they have all
+ *    disconnected.
+ *
+ * 2. **Switchover.** The VS's `DrainHandler` opens a second `/subscribe`
+ *    with `switchover: true` at the watermark of its last committed
+ *    transaction, which is routed to some RM. The VS continues to consume
+ *    the old RM's stream. If the RM is compatible (i.e. another old RM),
+ *    the subscription is accepted and the VS simply moves to it. If the RM
+ *    is the new one, the subscriber's replica version is older, and the
+ *    subscription proceeds to step 3.
+ *
+ * 3. **Queued.** `ChangeStreamerImpl.subscribe()` holds the subscription
+ *    (it is not added to the forwarder) and calls {@link request}. The
+ *    request is `queued` and `onQueued(position)` sends
+ *    `status {restore: {state: 'queued', position}}`, again whenever the
+ *    position changes. If the held connection closes while queued (e.g.
+ *    the VS's old stream ended and it gave up on the switchover), the
+ *    ticket is `release()`d and the request leaves the queue.
+ *
+ * 4. **Granted.** When the number in flight (`replacing` + `granted`) is
+ *    below `maxConcurrentRestores`, the head of the queue is `granted`: a
+ *    {@link Grant} is added (counting toward capacity), and `onGranted()`
+ *    sends `status {restore: {state: 'granted'}}`. The VS keeps the granted
+ *    connection open, and initiates a self-drain (`drainRequested`): it
+ *    fails its health checks, drains its clients, and keeps replicating
+ *    from the old RM in the meantime.
+ *
+ * 5. **Restart.** The VS process exits, closing the granted connection.
+ *    The ticket is `release()`d, marking the grant as `released` and
+ *    starting the pending grant TTL ({@link DEFAULT_PENDING_GRANT_TTL_MS}),
+ *    within which a replacement reservation is expected. The drain itself
+ *    does not count against the TTL.
+ *
+ * 6. **Reservation opened.** The restarted VS sends
+ *    `reserve-snapshot {taskID, localReplica}`, and
+ *    `ChangeStreamerImpl.startSnapshotReservation()` calls
+ *    {@link reservationOpened}:
+ *    * A `localReplica` of a different replica version (e.g. `R1`) is
+ *      `replacing`: the reservation is counted, and consumes an outstanding
+ *      grant (preferring a `released` one), so the number in flight is
+ *      unchanged.
+ *    * A `localReplica` of the same replica version is `unclassified` until
+ *      the reservation is confirmed ({@link reservationConfirmed}) with the
+ *      backup's minimum watermark. It is `replacing` if its watermark is
+ *      older (i.e. it cannot catch up from the change log).
+ *    * No `localReplica` (an empty disk) is not counted.
+ *
+ *    If the VS reached a different RM instead, the grant here expires after
+ *    the TTL and frees its capacity, while the replacement is counted (as
+ *    an ungranted one) by the RM that it reached.
+ *
+ * 7. **Restoring.** The VS restores the backup while holding the
+ *    reservation's connection open. A retried reservation from the same
+ *    task that arrives while the previous one is still open replaces it
+ *    (`SnapshotReservations` `onClose`) but does not end it (`onEnd`), so it
+ *    remains `replacing`. If the reservation instead closes without
+ *    subscribing (e.g. a failed or disconnected restore, after which the VS
+ *    retries), {@link reservationClosed} converts its capacity into a
+ *    `released` grant, which the retry's replacement reservation consumes
+ *    (step 6), or which expires after the pending grant TTL (e.g. if the VS
+ *    was deleted).
+ *
+ * 8. **Subscribed.** Once restored, the VS subscribes on the same
+ *    connection at `R2`, which is accepted. `ChangeStreamerImpl.subscribe()`
+ *    calls {@link reservationSubscribed}, which moves the replacement from
+ *    its reservation (the key leaves `replacing`) to the subscription
+ *    (`catchingUp`), and then closes the task's reservation. Its `onEnd`
+ *    calls {@link reservationClosed}, which is a no-op for the moved key.
+ *
+ * 9. **Caught up.** The VS catches up on the change stream from the
+ *    backup's watermark. Once it has acked all changes up to the head of
+ *    the stream (`Subscriber.caughtUp`), the replacement is released after
+ *    the readiness grace period, which covers the time for the VS to start
+ *    serving, and the next queued request is granted. If the VS disconnects
+ *    first, the replacement is released immediately.
+ *
+ * A VS that restarts outside of a switchover with an incompatible replica
+ * (e.g. after a crash) enters at step 6 without a grant, and its restore
+ * consumes capacity all the same, delaying queued requests.
+ *
+ * ## States
+ *
+ * | Term          | Where                    | Meaning                          |
+ * | ------------- | ------------------------ | -------------------------------- |
+ * | queued        | `Entry.state`            | Held, waiting for capacity       |
+ * | granted       | `Entry.state`, `#grants` | Permitted to restore, pending a  |
+ * |               |                          | replacement reservation          |
+ * | released      | `Entry.state`            | The held connection closed       |
+ * | released      | `Grant.released`         | The granted connection, or an    |
+ * |               |                          | unsubscribed replacement         |
+ * |               |                          | reservation, closed; the TTL is  |
+ * |               |                          | running                          |
+ * | unclassified  | `#unclassified`          | Same-version reservation, not    |
+ * |               |                          | yet confirmed                    |
+ * | replacing     | `#replacing`             | Open replacement reservation     |
+ * | catching up   | `#catchingUp`            | Subscribed replacement, not yet  |
+ * |               |                          | caught up (or within the         |
+ * |               |                          | readiness grace period)          |
+ * | in flight     | `#inFlight`              | `replacing` + `catchingUp` +     |
+ * |               |                          | `granted`, bounded by            |
+ * |               |                          | `maxConcurrentRestores`          |
+ */
+export declare class RestoreAdmission {
+    #private;
+    constructor(lc: LogContext, opts: RestoreAdmissionOptions);
+    /** For testing and logging. */
+    get stats(): {
+        replacing: number;
+        granted: number;
+        queued: number;
+    };
+    /**
+     * Called when a snapshot reservation is opened by a subscriber with the
+     * given `localReplica` (if any). A subscriber with a local replica of a
+     * different replica version is replacing it. One with the same replica
+     * version is replacing it if it is outdated, which is determined by
+     * {@link reservationConfirmed}. Idempotent for the same `key` (e.g. a
+     * retried reservation).
+     */
+    reservationOpened(key: string, localReplica: LocalReplica | undefined): void;
+    /**
+     * Called when a reservation is confirmed with the minimum watermark of the
+     * replica to restore.
+     */
+    reservationConfirmed(key: string, minWatermark: string): void;
+    /**
+     * Called when a reservation ends. A replacement reservation normally ends
+     * after its task subscribes ({@link reservationSubscribed}), in which case
+     * this is a no-op. A replacement reservation that ends without subscribing
+     * (e.g. a failed or disconnected restore) is most likely followed by a
+     * retry, so its capacity is retained as a released grant (which the retry
+     * consumes) rather than released immediately.
+     */
+    reservationClosed(key: string): void;
+    /**
+     * Called when the task of a reservation subscribes (before the reservation
+     * is closed). If the reservation is a replacement, it continues to be
+     * counted until the readiness grace period after the subscriber has caught
+     * up, or until the subscription closes.
+     */
+    reservationSubscribed(key: string): Replacement | undefined;
+    request(request: AdmissionRequest): AdmissionTicket;
+}
+//# sourceMappingURL=restore-admission.d.ts.map

package/out/zero-cache/src/services/change-streamer/restore-admission.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"restore-admission.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/restore-admission.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAEjD;;;;;;;;GAQG;AACH,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD;;;;;GAKG;AACH,eAAO,MAAM,iCAAiC,OAAQ,CAAC;AAEvD,MAAM,MAAM,uBAAuB,GAAG;IACpC,sDAAsD;IACtD,cAAc,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC;;;;;;;OAOG;IACH,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5C,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IAC7C,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;OAGG;IACH,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IAErC,iDAAiD;IACjD,SAAS,EAAE,MAAM,IAAI,CAAC;CACvB,CAAC;AAEF,4FAA4F;AAC5F,MAAM,MAAM,WAAW,GAAG;IACxB;;;OAGG;IACH,QAAQ,EAAE,MAAM,IAAI,CAAC;IAErB,wEAAwE;IACxE,MAAM,EAAE,MAAM,IAAI,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B;;;;;OAKG;IACH,OAAO,IAAI,IAAI,CAAC;CACjB,CAAC;AAeF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoIG;AACH,qBAAa,gBAAgB;;IAwB3B,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,uBAAuB,EAqBxD;IAMD,+BAA+B;IAC/B,IAAI,KAAK;QAEL,SAAS;QACT,OAAO;QACP,MAAM;MAET;IAED;;;;;;;OAOG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,GAAG,SAAS,QAUpE;IAED;;;OAGG;IACH,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,QAMrD;IAED;;;;;;;OAOG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,QAO5B;IAED;;;;;OAKG;IACH,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAwB1D;IAiBD,OAAO,CAAC,OAAO,EAAE,gBAAgB,GAAG,eAAe,CAKlD;CAoDF"}