@rocicorp/zero 1.12.0-canary.1 → 1.12.0-canary.11

package/out/zero-cache/src/services/change-streamer/subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#wsBatched","#backlogBackpressure","#backlogFullWaiters","#onAck","#backfills","#onAligned","#onBackfillIgnored","#lagReportingDisabled","#caughtUp","#watermark","#acked","#backlog","#aligned","#bufferedBacklogBytes","#pushBacklog","#maybeWaitForBacklogSpace","#sendChange","#initialized","#trackBackfills","#sendPreSerializedDownstream","#sendDownstream","#initialize","#backlogDrain","#drainBacklog","#sendStringifiedDownstream","#advanceAck","#checkCaughtUp","#caughtUpWatermark","#pending","#pendingBytes","#processed","#backlogCount","#samples","#totalBufferedBytes","#missedLastTimeout","#laggingSinceMs","#backlogBytes","#resolveBacklogFullWaiters","#backlogInFlightBytes","#lowWaterBytes","#waiters"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n  wsBatched?: boolean | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n\n  /**\n   * The subscriber's pending backfills (protocol v8+), which are tracked\n   * over the changes sent to the subscriber.\n   */\n  backfills?: BackfillState | undefined;\n\n  /**\n   * Called once the subscriber has been sent all changes up to the head of\n   * the stream, i.e. its catchup and backlog have been flushed. From this\n   * point on, the subscriber's tracked {@link backfills} reflect the same\n   * position in the stream as that of the change-streamer.\n   */\n  onAligned?: ((subscriber: Subscriber) => void) | undefined;\n\n  /**\n   * Called when a `backfill` or `backfill-completed` message is ignored for\n   * some of the subscriber's pending backfill columns, after the subscriber\n   * is aligned.\n   */\n  onBackfillIgnored?: ((columns: string[]) => void) | undefined;\n\n  /**\n   * Resolves to whether lag reporting is disabled upstream. If so, a\n   * {@link LAG_REPORTING_DISABLED} status is sent once the subscriber is\n   * aligned, as a stand-in for the lag reports that a subscriber would\n   * otherwise use to determine that it is caught up (e.g. to advertise\n   * readiness to serve traffic).\n   */\n  lagReportingDisabled?: Promise<boolean> | undefined;\n\n  /**\n   * Whether the subscriber switches over to another replication-manager\n   * when this one drains (see `DrainTracker`).\n   */\n  drainable?: boolean | undefined;\n};\n\n/**\n * A status message queued in the backlog, so that it is sent downstream in\n * the order in which it was received relative to (backlogged) changes.\n */\ntype QueuedStatus = {readonly status: string};\n\ntype BacklogEntry = WatermarkedChange | QueuedStatus;\n\nfunction isQueuedStatus(entry: BacklogEntry): entry is QueuedStatus {\n  return !Array.isArray(entry);\n}\n\nfunction entryBytes(entry: BacklogEntry): number {\n  return isQueuedStatus(entry) ? entry.status.length : entry[2].length;\n}\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly drainable: boolean;\n  readonly #downstream: Subscription<string | PreSerializedBatch>;\n  readonly #wsBatched: boolean;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<BacklogEntry> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n  readonly #backfills: BackfillState | undefined;\n  readonly #onAligned: ((subscriber: Subscriber) => void) | undefined;\n  readonly #onBackfillIgnored: ((columns: string[]) => void) | undefined;\n  readonly #lagReportingDisabled: Promise<boolean> | undefined;\n  #aligned = false;\n  // The watermark to be acked for the subscriber to be caught up, set when\n  // it is aligned.\n  #caughtUpWatermark: string | undefined;\n  readonly #caughtUp = resolver<void>();\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string | PreSerializedBatch>,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.drainable = options.drainable ?? false;\n    this.#downstream = downstream;\n    this.#wsBatched = options.wsBatched ?? false;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n    this.#backfills = options.backfills;\n    this.#onAligned = options.onAligned;\n    this.#onBackfillIgnored = options.onBackfillIgnored;\n    this.#lagReportingDisabled = options.lagReportingDisabled;\n  }\n\n  /**\n   * The subscriber's tracked pending backfills, or `undefined` if the\n   * subscriber does not report them (i.e. protocol < v8).\n   */\n  get backfills(): BackfillState | undefined {\n    return this.#backfills;\n  }\n\n  /**\n   * Whether the subscriber has been sent all changes up to the head of the\n   * stream (see {@link SubscriberOptions.onAligned}).\n   */\n  get aligned(): boolean {\n    return this.#aligned;\n  }\n\n  /**\n   * Resolves once the subscriber has caught up, i.e. once it has acked\n   * (durably applied) all changes up to the head of the stream at the time\n   * it was {@link aligned}. Never resolves if the subscriber is closed first.\n   */\n  get caughtUp(): Promise<void> {\n    return this.#caughtUp.promise;\n  }\n\n  #trackBackfills(change: WatermarkedChange) {\n    if (this.#backfills) {\n      const ignored = this.#backfills.applySerialized(change);\n      if (ignored.length && this.#aligned) {\n        this.#onBackfillIgnored?.(ignored);\n      }\n    }\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  sendBatch(\n    changes: readonly WatermarkedChange[],\n    preSerialized?: PreSerializedBatch | undefined,\n  ): Promise<void> {\n    if (changes.length === 0) {\n      return promiseVoid;\n    }\n\n    if (\n      this.#wsBatched &&\n      preSerialized &&\n      !this.#backlog &&\n      this.#initialized &&\n      changes[0][0] > this.#watermark &&\n      (this.#protocolVersion >= 5 ||\n        changes.every(c => this.supportsMessage(c[1])))\n    ) {\n      let commitWatermark: string | undefined;\n      for (let i = changes.length - 1; i >= 0; i--) {\n        if (changes[i][1] === 'commit') {\n          commitWatermark = changes[i][0];\n          break;\n        }\n      }\n      if (commitWatermark) {\n        this.#watermark = commitWatermark;\n      }\n      for (const change of changes) {\n        this.#trackBackfills(change);\n      }\n      return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n    }\n\n    const promises: Promise<void>[] = [];\n    for (const change of changes) {\n      const p = this.send(change);\n      if (p !== promiseVoid) {\n        promises.push(p);\n      }\n    }\n    if (promises.length === 0) {\n      return promiseVoid;\n    }\n    if (promises.length === 1) {\n      return promises[0];\n    }\n    return Promise.all(promises).then(() => {});\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      // The initial status precedes the catchup, so it bypasses the backlog.\n      void this.#sendDownstream(['status', {tag: 'status'}]);\n    }\n  }\n\n  /**\n   * Sends a status message (e.g. a lag report) downstream. While the\n   * subscriber is catching up, the status is queued in the backlog behind\n   * the changes that preceded it in the replication stream, so that the\n   * subscriber processes it in stream order. This is necessary for lag\n   * reports to measure the lag of the subscriber, including its catchup.\n   *\n   * Status messages received before the subscriber is initialized are\n   * dropped.\n   */\n  sendStatus(status: Status) {\n    if (!this.#initialized) {\n      return;\n    }\n    const downstream: Downstream = ['status', status];\n    if (this.#backlog) {\n      this.#pushBacklog({status: BigIntJSON.stringify(downstream)});\n      return;\n    }\n    void this.#sendDownstream(downstream);\n  }\n\n  /**\n   * Sends a control status (e.g. `draining` or `restore`) downstream\n   * immediately, regardless of whether the subscriber is initialized or\n   * catching up. Unlike {@link sendStatus}, such statuses are not tied to a\n   * position in the replication stream.\n   */\n  sendControlStatus(status: Status) {\n    void this.#sendDownstream(['status', status]);\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    this.#trackBackfills(change);\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      this.#advanceAck(watermark);\n    }\n  }\n\n  #advanceAck(watermark: string) {\n    // Sends can complete out of order (e.g. the bounded window in\n    // #drainBacklog), so the ack only advances monotonically, and listeners\n    // are only notified when it does.\n    const acked = max(this.#acked, watermark);\n    if (acked !== this.#acked) {\n      this.#acked = acked;\n      this.#onAck?.(acked);\n      this.#checkCaughtUp();\n    }\n  }\n\n  #checkCaughtUp() {\n    if (\n      this.#caughtUpWatermark !== undefined &&\n      this.#acked >= this.#caughtUpWatermark\n    ) {\n      this.#caughtUpWatermark = undefined;\n      this.#caughtUp.resolve();\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  async #sendPreSerializedDownstream(\n    batch: PreSerializedBatch,\n    commitWatermark?: string | undefined,\n  ): Promise<void> {\n    const size = batch.byteLength;\n    this.#pending += batch.changes.length;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(batch);\n    try {\n      const outcome = await result;\n      if (commitWatermark && outcome === 'consumed') {\n        this.#advanceAck(commitWatermark);\n      }\n    } finally {\n      this.#pending -= batch.changes.length;\n      this.#pendingBytes -= size;\n      this.#processed += batch.changes.length;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(\n    error?: ErrorType,\n    message?: string,\n    details?: {replicaVersion?: string | undefined},\n  ) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream([\n        'error',\n        {type: error, message, ...details},\n      ]).finally(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(entry: BacklogEntry) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(entry);\n    this.#backlogBytes += entryBytes(entry);\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const entry = this.#backlog?.shift();\n        if (!entry) {\n          const closed = this.#backlog === null;\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          if (!closed && !this.#aligned) {\n            // All changes up to the head have been handed to #sendChange()\n            // (and thus tracked), and subsequent changes are sent directly.\n            this.#aligned = true;\n            this.#onAligned?.(this);\n            this.#caughtUpWatermark = this.#watermark;\n            this.#checkCaughtUp();\n            void this.#lagReportingDisabled?.then(disabled => {\n              if (disabled) {\n                this.sendStatus(LAG_REPORTING_DISABLED);\n              }\n            });\n          }\n          break;\n        }\n\n        const bytes = entryBytes(entry);\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const send = isQueuedStatus(entry)\n          ? this.#sendStringifiedDownstream(entry.status).then(() => {})\n          : this.#sendChange(entry);\n        const promise = send.finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n\n/**\n * Sent to an aligned subscriber when lag reporting is disabled upstream,\n * signaling that the subscriber is caught up. This is a lag report with a\n * `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured\n * (see `isLagReportingDisabledSignal()` in the replicator's recorder).\n */\nexport const LAG_REPORTING_DISABLED: Status = {\n  tag: 'status',\n  lagReport: {nextSendTimeMs: 0},\n};\n"],"mappings":";;;;;;;;AAsBA,IAAM,mCAAmC,KAAK,OAAO;AACrD,IAAM,kCAAkC;AA2DxC,SAAS,eAAe,OAA4C;CAClE,OAAO,CAAC,MAAM,QAAQ,KAAK;AAC7B;AAEA,SAAS,WAAW,OAA6B;CAC/C,OAAO,eAAe,KAAK,IAAI,MAAM,OAAO,SAAS,MAAM,GAAG;AAChE;;;;;;;;AAuBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CACA;CACA;CACA;CACA;CACA,WAAW;CAGX;CACA,YAAqB,SAAe;CAEpC,YACE,iBACA,IACA,MACA,WACA,YACA,UAA6B,CAAC,GAC9B;EACA,KAAKA,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAK,YAAY,QAAQ,aAAa;EACtC,KAAKC,cAAc;EACnB,KAAKC,aAAa,QAAQ,aAAa;EACvC,KAAKS,aAAa;EAClB,KAAKC,SAAS;EACd,KAAKC,WAAW,IAAI,WAAW;EAC/B,KAAKV,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAKE,SAAS,QAAQ;EACtB,KAAKC,aAAa,QAAQ;EAC1B,KAAKC,aAAa,QAAQ;EAC1B,KAAKC,qBAAqB,QAAQ;EAClC,KAAKC,wBAAwB,QAAQ;CACvC;;;;;CAMA,IAAI,YAAuC;EACzC,OAAO,KAAKH;CACd;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAKQ;CACd;;;;;;CAOA,IAAI,WAA0B;EAC5B,OAAO,KAAKJ,UAAU;CACxB;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAKJ,YAAY;GACnB,MAAM,UAAU,KAAKA,WAAW,gBAAgB,MAAM;GACtD,IAAI,QAAQ,UAAU,KAAKQ,UACzB,KAAKN,qBAAqB,OAAO;EAErC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAKG;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAKC;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAKG,yBAAyB,KAAKZ,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAKU,aAAa,QAAQ,KAAKE,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAKX,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAKA,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAKO,YAAY;GAC/B,IAAI,KAAKE,UAAU;IAIjB,KAAKG,aAAa,MAAM;IACxB,OAAO,KAAKC,0BAA0B;GACxC;GACA,OAAO,KAAKC,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAKhB,cACL,iBACA,CAAC,KAAKW,YACN,KAAKM,gBACL,QAAQ,GAAG,KAAK,KAAKR,eACpB,KAAKX,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,GAAG,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,GAAG;IAC7B;GACF;GAEF,IAAI,iBACF,KAAKW,aAAa;GAEpB,KAAK,MAAM,UAAU,SACnB,KAAKS,gBAAgB,MAAM;GAE7B,OAAO,KAAKC,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,EAAE,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAKF,cAAc;GACtB,KAAKA,eAAe;GAEpB,KAAUG,gBAAgB,CAAC,UAAU,EAAC,KAAK,SAAQ,CAAC,CAAC;EACvD;CACF;;;;;;;;;;;CAYA,WAAW,QAAgB;EACzB,IAAI,CAAC,KAAKH,cACR;EAEF,MAAM,aAAyB,CAAC,UAAU,MAAM;EAChD,IAAI,KAAKN,UAAU;GACjB,KAAKG,aAAa,EAAC,QAAQ,WAAW,UAAU,UAAU,EAAC,CAAC;GAC5D;EACF;EACA,KAAUM,gBAAgB,UAAU;CACtC;;;;;;;CAQA,kBAAkB,QAAgB;EAChC,KAAUA,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAC9C;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAKC,YAAY;EACjB,MAAM,KAAKL,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAKK,YAAY;EACjB,IAAI,CAAC,KAAKV,UACR,OAAO,KAAKW,iBAAiB;EAE/B,IAAI,CAAC,KAAKA,eAAe;GAKvB,KAAKA,gBAAgB,KAAKC,cAAc;GACxC,KAAUD,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAKA;CACd;CAEA,MAAMN,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAKP,aAAa;EAEpB,KAAKS,gBAAgB,MAAM;EAC3B,MAAM,SAAS,MAAM,KAAKM,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YACjC,KAAKC,YAAY,SAAS;CAE9B;CAEA,YAAY,WAAmB;EAI7B,MAAM,QAAQ,IAAI,KAAKf,QAAQ,SAAS;EACxC,IAAI,UAAU,KAAKA,QAAQ;GACzB,KAAKA,SAAS;GACd,KAAKP,SAAS,KAAK;GACnB,KAAKuB,eAAe;EACtB;CACF;CAEA,iBAAiB;EACf,IACE,KAAKC,uBAAuB,KAAA,KAC5B,KAAKjB,UAAU,KAAKiB,oBACpB;GACA,KAAKA,qBAAqB,KAAA;GAC1B,KAAKnB,UAAU,QAAQ;EACzB;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAKgB,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAMA,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAKI;EACL,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK9B,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAK6B;GACL,KAAKC,iBAAiB;GACtB,KAAKC;EACP;CACF;CAEA,MAAMX,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAKS,YAAY,MAAM,QAAQ;EAC/B,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK9B,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YACjC,KAAK0B,YAAY,eAAe;EAEpC,UAAU;GACR,KAAKG,YAAY,MAAM,QAAQ;GAC/B,KAAKC,iBAAiB;GACtB,KAAKC,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAKF,WAAW,KAAKG;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAKD;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAKE,SAAS,UAAU,YAC7B,KAAKA,SAAS,MAAM;EAEtB,KAAKA,SAAS,KAAK;GAAC,WAAW,KAAKF;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAKE,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAKD;GACd,cAAc,KAAKlB;GACnB,oBAAoB,KAAKoB;GACzB,mBAAmB,KAAKC;EAC1B;EAEF,MAAM,OAAO,KAAKF,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAKA,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAKD;GACd,cAAc,KAAKlB;GACnB,oBAAoB,KAAKoB;GACzB,mBAAmB,KAAKC;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAKA,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAKC,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAKD,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAKC,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAKA,oBAAoB;EACzB,OAAO,MAAM,KAAKA;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAKrC,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MACE,OACA,SACA,SACA;EAGA,KAAKa,WAAW;EAChB,KAAKyB,gBAAgB;EACrB,KAAKnC,qBAAqB,WAAW;EAGrC,KAAKoC,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAUjB,gBAAgB,CACxB,SACA;GAAC,MAAM;GAAO;GAAS,GAAG;EAAO,CACnC,CAAC,EAAE,cAAc,KAAKrB,YAAY,OAAO,CAAC;OAE1C,KAAKA,YAAY,OAAO;CAE5B;CAEA,IAAIgC,gBAAgB;EAClB,OAAO,KAAKpB,UAAU,QAAQ;CAChC;CAEA,IAAIE,wBAAwB;EAI1B,OAAO,KAAKuB,gBAAgB,KAAKE;CACnC;CAEA,IAAIL,sBAAsB;EACxB,OAAO,KAAKpB,wBAAwB,KAAKgB;CAC3C;CAEA,aAAa,OAAqB;EAChC,OAAO,KAAKlB,UAAU,gDAAgD;EACtE,KAAKA,SAAS,KAAK,KAAK;EACxB,KAAKyB,iBAAiB,WAAW,KAAK;EACtC,IAAI,KAAK,aACP,KAAKC,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAKnC,mBAAmB;EAC5C,KAAKA,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAKD,qBAAqB,aAAa,KAAKY,qBAAqB;CAC1E;CAEA,MAAMU,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,QAAQ,KAAKZ,UAAU,MAAM;IACnC,IAAI,CAAC,OAAO;KACV,MAAM,SAAS,KAAKA,aAAa;KACjC,KAAKA,WAAW;KAChB,KAAKyB,gBAAgB;KACrB,KAAKnC,qBAAqB,uBACxB,KAAKY,qBACP;KACA,IAAI,CAAC,UAAU,CAAC,KAAKD,UAAU;MAG7B,KAAKA,WAAW;MAChB,KAAKP,aAAa,IAAI;MACtB,KAAKsB,qBAAqB,KAAKlB;MAC/B,KAAKiB,eAAe;MACpB,KAAUnB,uBAAuB,MAAK,aAAY;OAChD,IAAI,UACF,KAAK,WAAW,sBAAsB;MAE1C,CAAC;KACH;KACA;IACF;IAEA,MAAM,QAAQ,WAAW,KAAK;IAC9B,KAAK6B,iBAAiB;IACtB,KAAKE,yBAAyB;IAC9B,KAAKrC,qBAAqB,uBACxB,KAAKY,qBACP;IAQA,MAAM,WAHO,eAAe,KAAK,IAC7B,KAAKW,2BAA2B,MAAM,MAAM,EAAE,WAAW,CAAC,CAAC,IAC3D,KAAKR,YAAY,KAAK,GACL,cAAc;KACjC,KAAKsB,yBAAyB;KAC9B,KAAKrC,qBAAqB,uBACxB,KAAKY,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAKZ,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAKqB,gBAAgB;GACrB,KAAKrB,qBAAqB,uBACxB,KAAKY,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAK0B,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAKC,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAKA,SAAS,WAAW,KAAK,gBAAgB,KAAKD,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAKC,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF;;;;;;;AAQA,IAAa,yBAAiC;CAC5C,KAAK;CACL,WAAW,EAAC,gBAAgB,EAAC;AAC/B"}
+{"version":3,"file":"subscriber.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {resolver, type Resolver} from '../../../../shared/src/resolver.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n  wsBatched?: boolean | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n\n  /**\n   * The subscriber's pending backfills (protocol v8+), which are tracked\n   * over the changes sent to the subscriber.\n   */\n  backfills?: BackfillState | undefined;\n\n  /**\n   * Called once the subscriber has been sent all changes up to the head of\n   * the stream, i.e. its catchup and backlog have been flushed. From this\n   * point on, the subscriber's tracked {@link backfills} reflect the same\n   * position in the stream as that of the change-streamer.\n   */\n  onAligned?: ((subscriber: Subscriber) => void) | undefined;\n\n  /**\n   * Called when a `backfill` or `backfill-completed` message is ignored for\n   * some of the subscriber's pending backfill columns, after the subscriber\n   * is aligned.\n   */\n  onBackfillIgnored?: ((columns: string[]) => void) | undefined;\n\n  /**\n   * Resolves, once it is known whether lag reporting is enabled upstream, to\n   * the means of requesting a lag report, or to `undefined` if it is disabled.\n   *\n   * Subscribers use lag reports to determine that they are caught up (e.g. to\n   * advertise readiness to serve traffic). Once the subscriber is aligned, a\n   * report is therefore requested, so that it is received without waiting out\n   * the report interval. If lag reporting is disabled, a\n   * {@link LAG_REPORTING_DISABLED} status is sent as a stand-in instead.\n   */\n  lagReports?: Promise<LagReports | undefined> | undefined;\n\n  /**\n   * Whether the subscriber switches over to another replication-manager\n   * when this one drains (see `DrainTracker`).\n   */\n  drainable?: boolean | undefined;\n};\n\n/**\n * A status message queued in the backlog, so that it is sent downstream in\n * the order in which it was received relative to (backlogged) changes.\n */\ntype QueuedStatus = {readonly status: string};\n\ntype BacklogEntry = WatermarkedChange | QueuedStatus;\n\nfunction isQueuedStatus(entry: BacklogEntry): entry is QueuedStatus {\n  return !Array.isArray(entry);\n}\n\nfunction entryBytes(entry: BacklogEntry): number {\n  return isQueuedStatus(entry) ? entry.status.length : entry[2].length;\n}\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly drainable: boolean;\n  readonly #downstream: Subscription<string | PreSerializedBatch>;\n  readonly #wsBatched: boolean;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<BacklogEntry> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n  readonly #backfills: BackfillState | undefined;\n  readonly #onAligned: ((subscriber: Subscriber) => void) | undefined;\n  readonly #onBackfillIgnored: ((columns: string[]) => void) | undefined;\n  readonly #lagReports: Promise<LagReports | undefined> | undefined;\n  #aligned = false;\n  // The watermark to be acked for the subscriber to be caught up, set when\n  // it is aligned.\n  #caughtUpWatermark: string | undefined;\n  readonly #caughtUp = resolver<void>();\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string | PreSerializedBatch>,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.drainable = options.drainable ?? false;\n    this.#downstream = downstream;\n    this.#wsBatched = options.wsBatched ?? false;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n    this.#backfills = options.backfills;\n    this.#onAligned = options.onAligned;\n    this.#onBackfillIgnored = options.onBackfillIgnored;\n    this.#lagReports = options.lagReports;\n  }\n\n  /**\n   * The subscriber's tracked pending backfills, or `undefined` if the\n   * subscriber does not report them (i.e. protocol < v8).\n   */\n  get backfills(): BackfillState | undefined {\n    return this.#backfills;\n  }\n\n  /**\n   * Whether the subscriber has been sent all changes up to the head of the\n   * stream (see {@link SubscriberOptions.onAligned}).\n   */\n  get aligned(): boolean {\n    return this.#aligned;\n  }\n\n  /**\n   * Resolves once the subscriber has caught up, i.e. once it has acked\n   * (durably applied) all changes up to the head of the stream at the time\n   * it was {@link aligned}. Never resolves if the subscriber is closed first.\n   */\n  get caughtUp(): Promise<void> {\n    return this.#caughtUp.promise;\n  }\n\n  #trackBackfills(change: WatermarkedChange) {\n    if (this.#backfills) {\n      const ignored = this.#backfills.applySerialized(change);\n      if (ignored.length && this.#aligned) {\n        this.#onBackfillIgnored?.(ignored);\n      }\n    }\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  sendBatch(\n    changes: readonly WatermarkedChange[],\n    preSerialized?: PreSerializedBatch | undefined,\n  ): Promise<void> {\n    if (changes.length === 0) {\n      return promiseVoid;\n    }\n\n    if (\n      this.#wsBatched &&\n      preSerialized &&\n      !this.#backlog &&\n      this.#initialized &&\n      changes[0][0] > this.#watermark &&\n      (this.#protocolVersion >= 5 ||\n        changes.every(c => this.supportsMessage(c[1])))\n    ) {\n      let commitWatermark: string | undefined;\n      for (let i = changes.length - 1; i >= 0; i--) {\n        if (changes[i][1] === 'commit') {\n          commitWatermark = changes[i][0];\n          break;\n        }\n      }\n      if (commitWatermark) {\n        this.#watermark = commitWatermark;\n      }\n      for (const change of changes) {\n        this.#trackBackfills(change);\n      }\n      return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n    }\n\n    const promises: Promise<void>[] = [];\n    for (const change of changes) {\n      const p = this.send(change);\n      if (p !== promiseVoid) {\n        promises.push(p);\n      }\n    }\n    if (promises.length === 0) {\n      return promiseVoid;\n    }\n    if (promises.length === 1) {\n      return promises[0];\n    }\n    return Promise.all(promises).then(() => {});\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      // The initial status precedes the catchup, so it bypasses the backlog.\n      void this.#sendDownstream(['status', {tag: 'status'}]);\n    }\n  }\n\n  /**\n   * Sends a status message (e.g. a lag report) downstream. While the\n   * subscriber is catching up, the status is queued in the backlog behind\n   * the changes that preceded it in the replication stream, so that the\n   * subscriber processes it in stream order. This is necessary for lag\n   * reports to measure the lag of the subscriber, including its catchup.\n   *\n   * Status messages received before the subscriber is initialized are\n   * dropped.\n   */\n  sendStatus(status: Status) {\n    if (!this.#initialized) {\n      return;\n    }\n    const downstream: Downstream = ['status', status];\n    if (this.#backlog) {\n      this.#pushBacklog({status: BigIntJSON.stringify(downstream)});\n      return;\n    }\n    void this.#sendDownstream(downstream);\n  }\n\n  /**\n   * Sends a control status (e.g. `draining` or `restore`) downstream\n   * immediately, regardless of whether the subscriber is initialized or\n   * catching up. Unlike {@link sendStatus}, such statuses are not tied to a\n   * position in the replication stream.\n   */\n  sendControlStatus(status: Status) {\n    void this.#sendDownstream(['status', status]);\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    this.#trackBackfills(change);\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      this.#advanceAck(watermark);\n    }\n  }\n\n  #advanceAck(watermark: string) {\n    // Sends can complete out of order (e.g. the bounded window in\n    // #drainBacklog), so the ack only advances monotonically, and listeners\n    // are only notified when it does.\n    const acked = max(this.#acked, watermark);\n    if (acked !== this.#acked) {\n      this.#acked = acked;\n      this.#onAck?.(acked);\n      this.#checkCaughtUp();\n    }\n  }\n\n  #checkCaughtUp() {\n    if (\n      this.#caughtUpWatermark !== undefined &&\n      this.#acked >= this.#caughtUpWatermark\n    ) {\n      this.#caughtUpWatermark = undefined;\n      this.#caughtUp.resolve();\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  async #sendPreSerializedDownstream(\n    batch: PreSerializedBatch,\n    commitWatermark?: string | undefined,\n  ): Promise<void> {\n    const size = batch.byteLength;\n    this.#pending += batch.changes.length;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(batch);\n    try {\n      const outcome = await result;\n      if (commitWatermark && outcome === 'consumed') {\n        this.#advanceAck(commitWatermark);\n      }\n    } finally {\n      this.#pending -= batch.changes.length;\n      this.#pendingBytes -= size;\n      this.#processed += batch.changes.length;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(\n    error?: ErrorType,\n    message?: string,\n    details?: {replicaVersion?: string | undefined},\n  ) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream([\n        'error',\n        {type: error, message, ...details},\n      ]).finally(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(entry: BacklogEntry) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(entry);\n    this.#backlogBytes += entryBytes(entry);\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const entry = this.#backlog?.shift();\n        if (!entry) {\n          const closed = this.#backlog === null;\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          if (!closed && !this.#aligned) {\n            // All changes up to the head have been handed to #sendChange()\n            // (and thus tracked), and subsequent changes are sent directly.\n            this.#aligned = true;\n            this.#onAligned?.(this);\n            this.#caughtUpWatermark = this.#watermark;\n            this.#checkCaughtUp();\n            void this.#lagReports?.then(lagReports => {\n              if (lagReports) {\n                lagReports.request();\n              } else {\n                this.sendStatus(LAG_REPORTING_DISABLED);\n              }\n            });\n          }\n          break;\n        }\n\n        const bytes = entryBytes(entry);\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const send = isQueuedStatus(entry)\n          ? this.#sendStringifiedDownstream(entry.status).then(() => {})\n          : this.#sendChange(entry);\n        const promise = send.finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n\n/** Requests lag reports from upstream (see {@link SubscriberOptions.lagReports}). */\nexport type LagReports = {\n  /**\n   * Requests that a lag report be sent now rather than at the next report\n   * interval. Requests are coalesced upstream, e.g. with a report in flight.\n   */\n  request(): void;\n};\n\n/**\n * Sent to an aligned subscriber when lag reporting is disabled upstream,\n * signaling that the subscriber is caught up. This is a lag report with a\n * `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured\n * (see `isLagReportingDisabledSignal()` in the replicator's recorder).\n */\nexport const LAG_REPORTING_DISABLED: Status = {\n  tag: 'status',\n  lagReport: {nextSendTimeMs: 0},\n};\n"],"mappings":";;;;;;;;AAsBA,IAAM,mCAAmC;AACzC,IAAM,kCAAkC;AA8DxC,SAAS,eAAe,OAA4C;CAClE,OAAO,CAAC,MAAM,QAAQ,KAAK;AAC7B;AAEA,SAAS,WAAW,OAA6B;CAC/C,OAAO,eAAe,KAAK,IAAI,MAAM,OAAO,SAAS,MAAM,EAAE,CAAC;AAChE;;;;;;;;AAuBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CACA;CACA;CACA;CACA;CACA,WAAW;CAGX;CACA,YAAqB,SAAe;CAEpC,YACE,iBACA,IACA,MACA,WACA,YACA,UAA6B,CAAC,GAC9B;EACA,KAAK,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAK,YAAY,QAAQ,aAAa;EACtC,KAAK,cAAc;EACnB,KAAK,aAAa,QAAQ,aAAa;EACvC,KAAK,aAAa;EAClB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,WAAW;EAC/B,KAAK,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAK,SAAS,QAAQ;EACtB,KAAK,aAAa,QAAQ;EAC1B,KAAK,aAAa,QAAQ;EAC1B,KAAK,qBAAqB,QAAQ;EAClC,KAAK,cAAc,QAAQ;CAC7B;;;;;CAMA,IAAI,YAAuC;EACzC,OAAO,KAAK;CACd;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAK;CACd;;;;;;CAOA,IAAI,WAA0B;EAC5B,OAAO,KAAK,UAAU;CACxB;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAK,YAAY;GACnB,MAAM,UAAU,KAAK,WAAW,gBAAgB,MAAM;GACtD,IAAI,QAAQ,UAAU,KAAK,UACzB,KAAK,qBAAqB,OAAO;EAErC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAK;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAK;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAK,yBAAyB,KAAK,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAK,aAAa,QAAQ,KAAK,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAK,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAK,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAK,YAAY;GAC/B,IAAI,KAAK,UAAU;IAIjB,KAAK,aAAa,MAAM;IACxB,OAAO,KAAK,0BAA0B;GACxC;GACA,OAAO,KAAK,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAK,cACL,iBACA,CAAC,KAAK,YACN,KAAK,gBACL,QAAQ,EAAE,CAAC,KAAK,KAAK,eACpB,KAAK,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,EAAE,CAAC,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,EAAE,CAAC;IAC7B;GACF;GAEF,IAAI,iBACF,KAAK,aAAa;GAEpB,KAAK,MAAM,UAAU,SACnB,KAAK,gBAAgB,MAAM;GAE7B,OAAO,KAAK,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAK,cAAc;GACtB,KAAK,eAAe;GAEpB,KAAU,gBAAgB,CAAC,UAAU,EAAC,KAAK,SAAQ,CAAC,CAAC;EACvD;CACF;;;;;;;;;;;CAYA,WAAW,QAAgB;EACzB,IAAI,CAAC,KAAK,cACR;EAEF,MAAM,aAAyB,CAAC,UAAU,MAAM;EAChD,IAAI,KAAK,UAAU;GACjB,KAAK,aAAa,EAAC,QAAQ,WAAW,UAAU,UAAU,EAAC,CAAC;GAC5D;EACF;EACA,KAAU,gBAAgB,UAAU;CACtC;;;;;;;CAQA,kBAAkB,QAAgB;EAChC,KAAU,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAC9C;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAK,YAAY;EACjB,MAAM,KAAK,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAK,YAAY;EACjB,IAAI,CAAC,KAAK,UACR,OAAO,KAAK,iBAAiB;EAE/B,IAAI,CAAC,KAAK,eAAe;GAKvB,KAAK,gBAAgB,KAAK,cAAc;GACxC,KAAU,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAK;CACd;CAEA,MAAM,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAK,aAAa;EAEpB,KAAK,gBAAgB,MAAM;EAC3B,MAAM,SAAS,MAAM,KAAK,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YACjC,KAAK,YAAY,SAAS;CAE9B;CAEA,YAAY,WAAmB;EAI7B,MAAM,QAAQ,IAAI,KAAK,QAAQ,SAAS;EACxC,IAAI,UAAU,KAAK,QAAQ;GACzB,KAAK,SAAS;GACd,KAAK,SAAS,KAAK;GACnB,KAAK,eAAe;EACtB;CACF;CAEA,iBAAiB;EACf,IACE,KAAK,uBAAuB,KAAA,KAC5B,KAAK,UAAU,KAAK,oBACpB;GACA,KAAK,qBAAqB,KAAA;GAC1B,KAAK,UAAU,QAAQ;EACzB;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAK,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAM,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAK;EACL,KAAK,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAK;GACL,KAAK,iBAAiB;GACtB,KAAK;EACP;CACF;CAEA,MAAM,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAK,YAAY,MAAM,QAAQ;EAC/B,KAAK,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YACjC,KAAK,YAAY,eAAe;EAEpC,UAAU;GACR,KAAK,YAAY,MAAM,QAAQ;GAC/B,KAAK,iBAAiB;GACtB,KAAK,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAK,WAAW,KAAK;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAK;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAK,SAAS,UAAU,YAC7B,KAAK,SAAS,MAAM;EAEtB,KAAK,SAAS,KAAK;GAAC,WAAW,KAAK;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAK,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAK;GACd,cAAc,KAAK;GACnB,oBAAoB,KAAK;GACzB,mBAAmB,KAAK;EAC1B;EAEF,MAAM,OAAO,KAAK,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAK,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAK;GACd,cAAc,KAAK;GACnB,oBAAoB,KAAK;GACzB,mBAAmB,KAAK;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAK,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAK,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAK,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAK,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAK,oBAAoB;EACzB,OAAO,MAAM,KAAK;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAK,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MACE,OACA,SACA,SACA;EAGA,KAAK,WAAW;EAChB,KAAK,gBAAgB;EACrB,KAAK,qBAAqB,WAAW;EAGrC,KAAK,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAU,gBAAgB,CACxB,SACA;GAAC,MAAM;GAAO;GAAS,GAAG;EAAO,CACnC,CAAC,CAAC,CAAC,cAAc,KAAK,YAAY,OAAO,CAAC;OAE1C,KAAK,YAAY,OAAO;CAE5B;CAEA,IAAI,gBAAgB;EAClB,OAAO,KAAK,UAAU,QAAQ;CAChC;CAEA,IAAI,wBAAwB;EAI1B,OAAO,KAAK,gBAAgB,KAAK;CACnC;CAEA,IAAI,sBAAsB;EACxB,OAAO,KAAK,wBAAwB,KAAK;CAC3C;CAEA,aAAa,OAAqB;EAChC,OAAO,KAAK,UAAU,gDAAgD;EACtE,KAAK,SAAS,KAAK,KAAK;EACxB,KAAK,iBAAiB,WAAW,KAAK;EACtC,IAAI,KAAK,aACP,KAAK,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAK,mBAAmB;EAC5C,KAAK,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAK,qBAAqB,aAAa,KAAK,qBAAqB;CAC1E;CAEA,MAAM,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,QAAQ,KAAK,UAAU,MAAM;IACnC,IAAI,CAAC,OAAO;KACV,MAAM,SAAS,KAAK,aAAa;KACjC,KAAK,WAAW;KAChB,KAAK,gBAAgB;KACrB,KAAK,qBAAqB,uBACxB,KAAK,qBACP;KACA,IAAI,CAAC,UAAU,CAAC,KAAK,UAAU;MAG7B,KAAK,WAAW;MAChB,KAAK,aAAa,IAAI;MACtB,KAAK,qBAAqB,KAAK;MAC/B,KAAK,eAAe;MACpB,KAAU,aAAa,MAAK,eAAc;OACxC,IAAI,YACF,WAAW,QAAQ;YAEnB,KAAK,WAAW,sBAAsB;MAE1C,CAAC;KACH;KACA;IACF;IAEA,MAAM,QAAQ,WAAW,KAAK;IAC9B,KAAK,iBAAiB;IACtB,KAAK,yBAAyB;IAC9B,KAAK,qBAAqB,uBACxB,KAAK,qBACP;IAQA,MAAM,WAHO,eAAe,KAAK,IAC7B,KAAK,2BAA2B,MAAM,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC,IAC3D,KAAK,YAAY,KAAK,EAAA,CACL,cAAc;KACjC,KAAK,yBAAyB;KAC9B,KAAK,qBAAqB,uBACxB,KAAK,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAK,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAK,gBAAgB;GACrB,KAAK,qBAAqB,uBACxB,KAAK,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAK,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAK,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAK,SAAS,WAAW,KAAK,gBAAgB,KAAK,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAK,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF;;;;;;;AAiBA,IAAa,yBAAiC;CAC5C,KAAK;CACL,WAAW,EAAC,gBAAgB,EAAC;AAC/B"}

package/out/zero-cache/src/services/change-streamer/upstream-acker.js

@@ -69,7 +69,6 @@ var UpstreamAcker = class {
 			case "commit":
 				this.#lastTx = max(this.#lastTx, downstream[2].watermark);
 				this.#maybeAck();
-				break;
 		}
 	}
 	trackPgChangeLog(committedWatermark) {

package/out/zero-cache/src/services/change-streamer/upstream-acker.js.map

@@ -1 +1 @@
-{"version":3,"file":"upstream-acker.js","names":["#trackPgChangeLog","#trackBackup","#upstream","#lastTx","#lastStatus","#lastAck","#pgChangeLogWatermark","#maybeAck","#backupWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/upstream-acker.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport {max, min} from '../../types/lexi-version.ts';\nimport type {Sink} from '../../types/streams.ts';\nimport type {\n  ChangeSourceUpstream,\n  ChangeStreamMessage,\n} from '../change-source/protocol/current.ts';\n\ntype Opts = {\n  trackPgChangeLog: boolean;\n  trackBackup: boolean;\n};\n\n/**\n * Tracks the progress (watermark) of multiple streams:\n * - downstream transactions and status messages (the latter\n *   indicating LSNs that are not relevant to the publication\n *   but nevertheless need to be ACKed)\n * - PG change-log commits\n * - backup watermarks\n *\n * and sends upstream ACKs accordingly. When both the PG change-log\n * and backup watermarks are being considered for upstream ACKs\n * (i.e. RMv1.5), only watermarks that have been reached by both\n * stores are acked.\n *\n * The UpstreamAcker also takes into account that an upstream\n * connection can be disconnected and {@link reset()}. In this case,\n * the progress of the persistent stores is retained and the acks\n * are resent on the new upstream connection as necessary.\n */\nexport class UpstreamAcker {\n  readonly #trackPgChangeLog: boolean;\n  readonly #trackBackup: boolean;\n\n  #pgChangeLogWatermark = '';\n  #backupWatermark = '';\n\n  #upstream: Sink<ChangeSourceUpstream> | undefined;\n  #lastTx = '';\n  #lastStatus = '';\n  #lastAck = '';\n\n  constructor({trackPgChangeLog, trackBackup}: Opts) {\n    assert(\n      trackPgChangeLog || trackBackup,\n      `At least one of trackPgChangeLog or trackBackup must be true`,\n    );\n    this.#trackPgChangeLog = trackPgChangeLog;\n    this.#trackBackup = trackBackup;\n  }\n\n  /**\n   * Starts tracking a new upstream connection, which resumes the change stream\n   * after `resumeWatermark`.\n   *\n   * Everything at or before `resumeWatermark` was committed on an earlier\n   * connection, so it counts as committed on this one: a status watermark past\n   * it waits for the tracked stores to reach it, as it would for a commit on\n   * this connection. Otherwise the first keepalive of a stream that resumes\n   * ahead of the stores -- a change log ahead of its backup, which is the\n   * normal state -- would move the replication slot past transactions that the\n   * backup has not persisted, and a task restored from the backup would then\n   * resume below the slot, which the upstream moves forward without a word.\n   *\n   * When the PG change log is tracked, the stream resumes from its head, so\n   * `resumeWatermark` has been persisted in it (by this or another\n   * change-streamer). Without this, status messages would not be acked on an\n   * idle upstream until a commit is stored.\n   */\n  reset(upstream: Sink<ChangeSourceUpstream>, resumeWatermark: string) {\n    this.#upstream = upstream;\n    this.#lastTx = resumeWatermark;\n    this.#lastStatus = '';\n    this.#lastAck = '';\n    if (this.#trackPgChangeLog) {\n      this.#pgChangeLogWatermark = max(\n        resumeWatermark,\n        this.#pgChangeLogWatermark,\n      );\n    }\n    // Re-ack what the stores have persisted on the new connection.\n    this.#maybeAck();\n  }\n\n  trackDownstream(downstream: ChangeStreamMessage) {\n    const [tag, msg] = downstream;\n    switch (tag) {\n      case 'status':\n        if (msg.ack) {\n          this.#lastStatus = downstream[2].watermark;\n        }\n        this.#maybeAck();\n        break;\n      case 'commit':\n        // A commit replayed from below the resume watermark does not lower it.\n        this.#lastTx = max(this.#lastTx, downstream[2].watermark);\n        this.#maybeAck();\n        break;\n    }\n  }\n\n  trackPgChangeLog(committedWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#pgChangeLogWatermark = max(\n      committedWatermark,\n      this.#pgChangeLogWatermark,\n    );\n    this.#maybeAck();\n  }\n\n  trackBackup(backedUpWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#backupWatermark = max(backedUpWatermark, this.#backupWatermark);\n    this.#maybeAck();\n  }\n\n  #maybeAck() {\n    const currentWatermark = !this.#trackBackup\n      ? this.#pgChangeLogWatermark\n      : !this.#trackPgChangeLog\n        ? this.#backupWatermark\n        : min(this.#pgChangeLogWatermark, this.#backupWatermark);\n    if (currentWatermark > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {tag: 'commit'},\n        {watermark: currentWatermark},\n      ]);\n      this.#lastAck = currentWatermark;\n    }\n    // If all committed transactions have been acked, ack any outstanding\n    // status LSN's thereafter (i.e. non-publication upstream changes).\n    if (this.#lastAck >= this.#lastTx && this.#lastStatus > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {ack: true},\n        {watermark: this.#lastStatus},\n      ]);\n      this.#lastAck = this.#lastStatus;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+BA,IAAa,gBAAb,MAA2B;CACzB;CACA;CAEA,wBAAwB;CACxB,mBAAmB;CAEnB;CACA,UAAU;CACV,cAAc;CACd,WAAW;CAEX,YAAY,EAAC,kBAAkB,eAAoB;EACjD,OACE,oBAAoB,aACpB,8DACF;EACA,KAAKA,oBAAoB;EACzB,KAAKC,eAAe;CACtB;;;;;;;;;;;;;;;;;;;CAoBA,MAAM,UAAsC,iBAAyB;EACnE,KAAKC,YAAY;EACjB,KAAKC,UAAU;EACf,KAAKC,cAAc;EACnB,KAAKC,WAAW;EAChB,IAAI,KAAKL,mBACP,KAAKM,wBAAwB,IAC3B,iBACA,KAAKA,qBACP;EAGF,KAAKC,UAAU;CACjB;CAEA,gBAAgB,YAAiC;EAC/C,MAAM,CAAC,KAAK,OAAO;EACnB,QAAQ,KAAR;GACE,KAAK;IACH,IAAI,IAAI,KACN,KAAKH,cAAc,WAAW,GAAG;IAEnC,KAAKG,UAAU;IACf;GACF,KAAK;IAEH,KAAKJ,UAAU,IAAI,KAAKA,SAAS,WAAW,GAAG,SAAS;IACxD,KAAKI,UAAU;IACf;EACJ;CACF;CAEA,iBAAiB,oBAA4B;EAE3C,KAAKD,wBAAwB,IAC3B,oBACA,KAAKA,qBACP;EACA,KAAKC,UAAU;CACjB;CAEA,YAAY,mBAA2B;EAErC,KAAKC,mBAAmB,IAAI,mBAAmB,KAAKA,gBAAgB;EACpE,KAAKD,UAAU;CACjB;CAEA,YAAY;EACV,MAAM,mBAAmB,CAAC,KAAKN,eAC3B,KAAKK,wBACL,CAAC,KAAKN,oBACJ,KAAKQ,mBACL,IAAI,KAAKF,uBAAuB,KAAKE,gBAAgB;EAC3D,IAAI,mBAAmB,KAAKH,UAAU;GACpC,KAAKH,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,SAAQ;IACd,EAAC,WAAW,iBAAgB;GAC9B,CAAC;GACD,KAAKG,WAAW;EAClB;EAGA,IAAI,KAAKA,YAAY,KAAKF,WAAW,KAAKC,cAAc,KAAKC,UAAU;GACrE,KAAKH,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,KAAI;IACV,EAAC,WAAW,KAAKE,YAAW;GAC9B,CAAC;GACD,KAAKC,WAAW,KAAKD;EACvB;CACF;AACF"}
+{"version":3,"file":"upstream-acker.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/upstream-acker.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport {max, min} from '../../types/lexi-version.ts';\nimport type {Sink} from '../../types/streams.ts';\nimport type {\n  ChangeSourceUpstream,\n  ChangeStreamMessage,\n} from '../change-source/protocol/current.ts';\n\ntype Opts = {\n  trackPgChangeLog: boolean;\n  trackBackup: boolean;\n};\n\n/**\n * Tracks the progress (watermark) of multiple streams:\n * - downstream transactions and status messages (the latter\n *   indicating LSNs that are not relevant to the publication\n *   but nevertheless need to be ACKed)\n * - PG change-log commits\n * - backup watermarks\n *\n * and sends upstream ACKs accordingly. When both the PG change-log\n * and backup watermarks are being considered for upstream ACKs\n * (i.e. RMv1.5), only watermarks that have been reached by both\n * stores are acked.\n *\n * The UpstreamAcker also takes into account that an upstream\n * connection can be disconnected and {@link reset()}. In this case,\n * the progress of the persistent stores is retained and the acks\n * are resent on the new upstream connection as necessary.\n */\nexport class UpstreamAcker {\n  readonly #trackPgChangeLog: boolean;\n  readonly #trackBackup: boolean;\n\n  #pgChangeLogWatermark = '';\n  #backupWatermark = '';\n\n  #upstream: Sink<ChangeSourceUpstream> | undefined;\n  #lastTx = '';\n  #lastStatus = '';\n  #lastAck = '';\n\n  constructor({trackPgChangeLog, trackBackup}: Opts) {\n    assert(\n      trackPgChangeLog || trackBackup,\n      `At least one of trackPgChangeLog or trackBackup must be true`,\n    );\n    this.#trackPgChangeLog = trackPgChangeLog;\n    this.#trackBackup = trackBackup;\n  }\n\n  /**\n   * Starts tracking a new upstream connection, which resumes the change stream\n   * after `resumeWatermark`.\n   *\n   * Everything at or before `resumeWatermark` was committed on an earlier\n   * connection, so it counts as committed on this one: a status watermark past\n   * it waits for the tracked stores to reach it, as it would for a commit on\n   * this connection. Otherwise the first keepalive of a stream that resumes\n   * ahead of the stores -- a change log ahead of its backup, which is the\n   * normal state -- would move the replication slot past transactions that the\n   * backup has not persisted, and a task restored from the backup would then\n   * resume below the slot, which the upstream moves forward without a word.\n   *\n   * When the PG change log is tracked, the stream resumes from its head, so\n   * `resumeWatermark` has been persisted in it (by this or another\n   * change-streamer). Without this, status messages would not be acked on an\n   * idle upstream until a commit is stored.\n   */\n  reset(upstream: Sink<ChangeSourceUpstream>, resumeWatermark: string) {\n    this.#upstream = upstream;\n    this.#lastTx = resumeWatermark;\n    this.#lastStatus = '';\n    this.#lastAck = '';\n    if (this.#trackPgChangeLog) {\n      this.#pgChangeLogWatermark = max(\n        resumeWatermark,\n        this.#pgChangeLogWatermark,\n      );\n    }\n    // Re-ack what the stores have persisted on the new connection.\n    this.#maybeAck();\n  }\n\n  trackDownstream(downstream: ChangeStreamMessage) {\n    const [tag, msg] = downstream;\n    switch (tag) {\n      case 'status':\n        if (msg.ack) {\n          this.#lastStatus = downstream[2].watermark;\n        }\n        this.#maybeAck();\n        break;\n      case 'commit':\n        // A commit replayed from below the resume watermark does not lower it.\n        this.#lastTx = max(this.#lastTx, downstream[2].watermark);\n        this.#maybeAck();\n        break;\n    }\n  }\n\n  trackPgChangeLog(committedWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#pgChangeLogWatermark = max(\n      committedWatermark,\n      this.#pgChangeLogWatermark,\n    );\n    this.#maybeAck();\n  }\n\n  trackBackup(backedUpWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#backupWatermark = max(backedUpWatermark, this.#backupWatermark);\n    this.#maybeAck();\n  }\n\n  #maybeAck() {\n    const currentWatermark = !this.#trackBackup\n      ? this.#pgChangeLogWatermark\n      : !this.#trackPgChangeLog\n        ? this.#backupWatermark\n        : min(this.#pgChangeLogWatermark, this.#backupWatermark);\n    if (currentWatermark > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {tag: 'commit'},\n        {watermark: currentWatermark},\n      ]);\n      this.#lastAck = currentWatermark;\n    }\n    // If all committed transactions have been acked, ack any outstanding\n    // status LSN's thereafter (i.e. non-publication upstream changes).\n    if (this.#lastAck >= this.#lastTx && this.#lastStatus > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {ack: true},\n        {watermark: this.#lastStatus},\n      ]);\n      this.#lastAck = this.#lastStatus;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+BA,IAAa,gBAAb,MAA2B;CACzB;CACA;CAEA,wBAAwB;CACxB,mBAAmB;CAEnB;CACA,UAAU;CACV,cAAc;CACd,WAAW;CAEX,YAAY,EAAC,kBAAkB,eAAoB;EACjD,OACE,oBAAoB,aACpB,8DACF;EACA,KAAK,oBAAoB;EACzB,KAAK,eAAe;CACtB;;;;;;;;;;;;;;;;;;;CAoBA,MAAM,UAAsC,iBAAyB;EACnE,KAAK,YAAY;EACjB,KAAK,UAAU;EACf,KAAK,cAAc;EACnB,KAAK,WAAW;EAChB,IAAI,KAAK,mBACP,KAAK,wBAAwB,IAC3B,iBACA,KAAK,qBACP;EAGF,KAAK,UAAU;CACjB;CAEA,gBAAgB,YAAiC;EAC/C,MAAM,CAAC,KAAK,OAAO;EACnB,QAAQ,KAAR;GACE,KAAK;IACH,IAAI,IAAI,KACN,KAAK,cAAc,WAAW,EAAE,CAAC;IAEnC,KAAK,UAAU;IACf;GACF,KAAK;IAEH,KAAK,UAAU,IAAI,KAAK,SAAS,WAAW,EAAE,CAAC,SAAS;IACxD,KAAK,UAAU;EAEnB;CACF;CAEA,iBAAiB,oBAA4B;EAE3C,KAAK,wBAAwB,IAC3B,oBACA,KAAK,qBACP;EACA,KAAK,UAAU;CACjB;CAEA,YAAY,mBAA2B;EAErC,KAAK,mBAAmB,IAAI,mBAAmB,KAAK,gBAAgB;EACpE,KAAK,UAAU;CACjB;CAEA,YAAY;EACV,MAAM,mBAAmB,CAAC,KAAK,eAC3B,KAAK,wBACL,CAAC,KAAK,oBACJ,KAAK,mBACL,IAAI,KAAK,uBAAuB,KAAK,gBAAgB;EAC3D,IAAI,mBAAmB,KAAK,UAAU;GACpC,KAAK,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,SAAQ;IACd,EAAC,WAAW,iBAAgB;GAC9B,CAAC;GACD,KAAK,WAAW;EAClB;EAGA,IAAI,KAAK,YAAY,KAAK,WAAW,KAAK,cAAc,KAAK,UAAU;GACrE,KAAK,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,KAAI;IACV,EAAC,WAAW,KAAK,YAAW;GAC9B,CAAC;GACD,KAAK,WAAW,KAAK;EACvB;CACF;AACF"}

package/out/zero-cache/src/services/heapz.js

@@ -4,7 +4,8 @@ import v8 from "v8";
 import auth from "basic-auth";
 //#region ../zero-cache/src/services/heapz.ts
 function handleHeapzRequest(lc, config, req, res) {
-	if (!isAdminPasswordValid(lc, config, auth(req)?.pass)) {
+	const credentials = auth(req);
+	if (!isAdminPasswordValid(lc, config, credentials?.pass)) {
 		res.code(401).header("WWW-Authenticate", "Basic realm=\"Heapz Protected Area\"").send("Unauthorized");
 		return;
 	}

package/out/zero-cache/src/services/heapz.js.map

@@ -1 +1 @@
-{"version":3,"file":"heapz.js","names":[],"sources":["../../../../../zero-cache/src/services/heapz.ts"],"sourcesContent":["import fs from 'fs';\nimport v8 from 'v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\n\nexport function handleHeapzRequest(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Heapz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  const filename = v8.writeHeapSnapshot();\n  const stream = fs.createReadStream(filename);\n  void res\n    .header('Content-Type', 'application/octet-stream')\n    .header('Content-Disposition', `attachment; filename=${filename}`)\n    .send(stream);\n\n  // Clean up temp file after streaming\n  stream.on('end', () => {\n    fs.unlink(filename, err => {\n      if (err) {\n        lc.error?.('Error deleting heap snapshot:', err);\n      }\n    });\n  });\n}\n"],"mappings":";;;;;AAQA,SAAgB,mBACd,IACA,QACA,KACA,KACA;CAEA,IAAI,CAAC,qBAAqB,IAAI,QADV,KAAK,GACa,GAAa,IAAI,GAAG;EACxD,IACG,KAAK,GAAG,EACR,OAAO,oBAAoB,sCAAoC,EAC/D,KAAK,cAAc;EACtB;CACF;CAEA,MAAM,WAAW,GAAG,kBAAkB;CACtC,MAAM,SAAS,GAAG,iBAAiB,QAAQ;CAC3C,IACG,OAAO,gBAAgB,0BAA0B,EACjD,OAAO,uBAAuB,wBAAwB,UAAU,EAChE,KAAK,MAAM;CAGd,OAAO,GAAG,aAAa;EACrB,GAAG,OAAO,WAAU,QAAO;GACzB,IAAI,KACF,GAAG,QAAQ,iCAAiC,GAAG;EAEnD,CAAC;CACH,CAAC;AACH"}
+{"version":3,"file":"heapz.js","names":[],"sources":["../../../../../zero-cache/src/services/heapz.ts"],"sourcesContent":["import fs from 'fs';\nimport v8 from 'v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\n\nexport function handleHeapzRequest(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Heapz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  const filename = v8.writeHeapSnapshot();\n  const stream = fs.createReadStream(filename);\n  void res\n    .header('Content-Type', 'application/octet-stream')\n    .header('Content-Disposition', `attachment; filename=${filename}`)\n    .send(stream);\n\n  // Clean up temp file after streaming\n  stream.on('end', () => {\n    fs.unlink(filename, err => {\n      if (err) {\n        lc.error?.('Error deleting heap snapshot:', err);\n      }\n    });\n  });\n}\n"],"mappings":";;;;;AAQA,SAAgB,mBACd,IACA,QACA,KACA,KACA;CACA,MAAM,cAAc,KAAK,GAAG;CAC5B,IAAI,CAAC,qBAAqB,IAAI,QAAQ,aAAa,IAAI,GAAG;EACxD,IACG,KAAK,GAAG,CAAC,CACT,OAAO,oBAAoB,sCAAoC,CAAC,CAChE,KAAK,cAAc;EACtB;CACF;CAEA,MAAM,WAAW,GAAG,kBAAkB;CACtC,MAAM,SAAS,GAAG,iBAAiB,QAAQ;CAC3C,IACG,OAAO,gBAAgB,0BAA0B,CAAC,CAClD,OAAO,uBAAuB,wBAAwB,UAAU,CAAC,CACjE,KAAK,MAAM;CAGd,OAAO,GAAG,aAAa;EACrB,GAAG,OAAO,WAAU,QAAO;GACzB,IAAI,KACF,GAAG,QAAQ,iCAAiC,GAAG;EAEnD,CAAC;CACH,CAAC;AACH"}

package/out/zero-cache/src/services/http-service.d.ts

@@ -24,10 +24,16 @@ export declare class HttpService implements Service {
     run(): Promise<void>;
     /**
      * Responds to subsequent health checks (both "/" and "/keepalive") with a
-     * 503, e.g. to signal the container runner to stop routing to, and shut
-     * down, a server that is draining itself (see {@link SelfDrain}).
+     * 503, e.g. to signal the load balancer to stop routing to a server that
+     * is draining (see {@link SelfDrain}).
      */
     failHealthChecks(): void;
+    /**
+     * Responds to subsequent readiness checks ("/") with a 503, while liveness
+     * checks ("/keepalive") continue to succeed, e.g. to signal the load
+     * balancer to stop routing to a server that is about to drain.
+     */
+    failReadiness(): void;
     /**
      * Stops accepting new connections (e.g. when draining), while existing
      * connections, including websockets, continue to be served until

package/out/zero-cache/src/services/http-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IAGvC,aAAa,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAOxC,YACE,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAczD;IAGD,SAAS,CAAC,QAAQ,SAAK;IACvB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjC;IAGK,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAwB7B;IAEK,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzB;IAED;;;;OAIG;IACH,gBAAgB,SAGf;IAED;;;;OAIG;IACH,aAAa,SAKZ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAM1B;CACF"}
+{"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IAGvC,aAAa,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAQxC,YACE,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAczD;IAGD,SAAS,CAAC,QAAQ,SAAK;IACvB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjC;IAGK,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAwB7B;IAEK,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzB;IAED;;;;OAIG;IACH,gBAAgB,SAKf;IAED;;;;OAIG;IACH,aAAa,SAKZ;IAED;;;;OAIG;IACH,aAAa,SAKZ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAM1B;CACF"}

package/out/zero-cache/src/services/http-service.js

@@ -17,6 +17,7 @@ var HttpService = class {
 	#heartbeatMonitor;
 	#init;
 	#ready = false;
+	#failReadiness = false;
 	#failHealthChecks = false;
 	constructor(id, lc, opts, init) {
 		const { port, keepaliveTimeoutMs, readinessGate = promiseVoid } = opts;
@@ -35,7 +36,7 @@ var HttpService = class {
 	}
 	async start() {
 		this.#fastify.get("/", (_req, res) => {
-			if (this.#failHealthChecks) res.code(503).send("Draining");
+			if (this.#failReadiness || this.#failHealthChecks) res.code(503).send("Draining");
 			else if (this.#ready) res.send("OK");
 		});
 		this.#fastify.get("/keepalive", ({ headers }, res) => {
@@ -58,12 +59,25 @@ var HttpService = class {
 	}
 	/**
 	* Responds to subsequent health checks (both "/" and "/keepalive") with a
-	* 503, e.g. to signal the container runner to stop routing to, and shut
-	* down, a server that is draining itself (see {@link SelfDrain}).
+	* 503, e.g. to signal the load balancer to stop routing to a server that
+	* is draining (see {@link SelfDrain}).
 	*/
 	failHealthChecks() {
-		this._lc.info?.(`${this.id}: failing health checks to self drain`);
-		this.#failHealthChecks = true;
+		if (!this.#failHealthChecks) {
+			this._lc.info?.(`${this.id}: failing health checks while draining`);
+			this.#failHealthChecks = true;
+		}
+	}
+	/**
+	* Responds to subsequent readiness checks ("/") with a 503, while liveness
+	* checks ("/keepalive") continue to succeed, e.g. to signal the load
+	* balancer to stop routing to a server that is about to drain.
+	*/
+	failReadiness() {
+		if (!this.#failReadiness) {
+			this._lc.info?.(`${this.id}: failing readiness checks before draining`);
+			this.#failReadiness = true;
+		}
 	}
 	/**
 	* Stops accepting new connections (e.g. when draining), while existing

package/out/zero-cache/src/services/http-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.js","names":["#fastify","#port","#heartbeatMonitor","#init","#ready","#failHealthChecks"],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n\n  // Wait for the readinessGate to resolve before responding to health checks.\n  readinessGate?: Promise<unknown> | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  #ready = false;\n  #failHealthChecks = false;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n\n    void readinessGate.then(() => (this.#ready = true));\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => {\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  /**\n   * Responds to subsequent health checks (both \"/\" and \"/keepalive\") with a\n   * 503, e.g. to signal the container runner to stop routing to, and shut\n   * down, a server that is draining itself (see {@link SelfDrain}).\n   */\n  failHealthChecks() {\n    this._lc.info?.(`${this.id}: failing health checks to self drain`);\n    this.#failHealthChecks = true;\n  }\n\n  /**\n   * Stops accepting new connections (e.g. when draining), while existing\n   * connections, including websockets, continue to be served until\n   * {@link stop} is called.\n   */\n  stopAccepting() {\n    if (this.#fastify.server.listening) {\n      this._lc.info?.(`${this.id}: no longer accepting new connections`);\n      this.#fastify.server.close();\n    }\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CACT,oBAAoB;CAEpB,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKA,WAAW,QAAQ;EACxB,KAAKC,QAAQ;EACb,KAAKE,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAKD,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAKE,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAKJ,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAKK,mBACP,IAAI,KAAK,GAAG,EAAE,KAAK,UAAU;QACxB,IAAI,KAAKD,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAKJ,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAKE,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAKG,mBACP,IAAI,KAAK,GAAG,EAAE,KAAK,UAAU;QACxB,IAAI,KAAKD,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAKD,MAAM,KAAKH,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAKA,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAKC;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;;;;;;CAOA,mBAAmB;EACjB,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,sCAAsC;EACjE,KAAKI,oBAAoB;CAC3B;;;;;;CAOA,gBAAgB;EACd,IAAI,KAAKL,SAAS,OAAO,WAAW;GAClC,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,sCAAsC;GACjE,KAAKA,SAAS,OAAO,MAAM;EAC7B;CACF;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAKE,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAKF,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}
+{"version":3,"file":"http-service.js","names":[],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n\n  // Wait for the readinessGate to resolve before responding to health checks.\n  readinessGate?: Promise<unknown> | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  #ready = false;\n  #failReadiness = false;\n  #failHealthChecks = false;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n\n    void readinessGate.then(() => (this.#ready = true));\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => {\n      if (this.#failReadiness || this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  /**\n   * Responds to subsequent health checks (both \"/\" and \"/keepalive\") with a\n   * 503, e.g. to signal the load balancer to stop routing to a server that\n   * is draining (see {@link SelfDrain}).\n   */\n  failHealthChecks() {\n    if (!this.#failHealthChecks) {\n      this._lc.info?.(`${this.id}: failing health checks while draining`);\n      this.#failHealthChecks = true;\n    }\n  }\n\n  /**\n   * Responds to subsequent readiness checks (\"/\") with a 503, while liveness\n   * checks (\"/keepalive\") continue to succeed, e.g. to signal the load\n   * balancer to stop routing to a server that is about to drain.\n   */\n  failReadiness() {\n    if (!this.#failReadiness) {\n      this._lc.info?.(`${this.id}: failing readiness checks before draining`);\n      this.#failReadiness = true;\n    }\n  }\n\n  /**\n   * Stops accepting new connections (e.g. when draining), while existing\n   * connections, including websockets, continue to be served until\n   * {@link stop} is called.\n   */\n  stopAccepting() {\n    if (this.#fastify.server.listening) {\n      this._lc.info?.(`${this.id}: no longer accepting new connections`);\n      this.#fastify.server.close();\n    }\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CACT,iBAAiB;CACjB,oBAAoB;CAEpB,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAK,WAAW,QAAQ;EACxB,KAAK,QAAQ;EACb,KAAK,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAK,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAK,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAK,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAK,kBAAkB,KAAK,mBAC9B,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU;QACxB,IAAI,KAAK,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAK,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAK,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAK,mBACP,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU;QACxB,IAAI,KAAK,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAK,MAAM,KAAK,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAK,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAK;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;;;;;;CAOA,mBAAmB;EACjB,IAAI,CAAC,KAAK,mBAAmB;GAC3B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,uCAAuC;GAClE,KAAK,oBAAoB;EAC3B;CACF;;;;;;CAOA,gBAAgB;EACd,IAAI,CAAC,KAAK,gBAAgB;GACxB,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,2CAA2C;GACtE,KAAK,iBAAiB;EACxB;CACF;;;;;;CAOA,gBAAgB;EACd,IAAI,KAAK,SAAS,OAAO,WAAW;GAClC,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,sCAAsC;GACjE,KAAK,SAAS,OAAO,MAAM;EAC7B;CACF;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAK,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAK,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}

package/out/zero-cache/src/services/life-cycle.d.ts

@@ -1,3 +1,4 @@
+import { EventEmitter as NodeEventEmitter } from 'node:events';
 import type { IncomingHttpHeaders } from 'node:http';
 import type { EventEmitter } from 'stream';
 import type { LogContext } from '@rocicorp/logger';
@@ -94,6 +95,24 @@ export declare class SelfDrain {
     constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions);
     request(): void;
 }
+/**
+ * The event that a {@link PreDrain} emits immediately upon the first graceful
+ * shutdown signal, with that signal as its argument, before the delay.
+ */
+export declare const PRE_DRAIN = "preDrain";
+/**
+ * Delays graceful shutdown signals (`SIGTERM`, `SIGINT`), e.g. so that load
+ * balancers can observe a failed readiness check before the server begins
+ * to drain. The first graceful signal is announced immediately with a
+ * {@link PRE_DRAIN} event, and is re-emitted to listeners of the signal, such
+ * as those of the {@link ProcessManager} and {@link runUntilKilled}, once the
+ * delay has elapsed. Repeated graceful signals are dropped. All other events,
+ * including forceful signals, are forwarded immediately.
+ */
+export declare class PreDrain extends NodeEventEmitter {
+    #private;
+    constructor(lc: LogContext, proc: EventEmitter, delayMs: number);
+}
 /**
  * The maximum time that {@link runUntilKilled} waits for its services to stop
  * (i.e. for their `run()` to settle) after stopping them.

package/out/zero-cache/src/services/life-cycle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAG/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,QAAQ,CAAU,CAAC;AAChE,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,SAAS,CAAU,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;IAWzB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAyC7C;IAEK,QAAQ,kBAKb;IAuBD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,QAsB7D;IAKD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAwBhE;IAED,YAAY,IAAI,MAAM,EAAE,CAEvB;IAEK,eAAe,kBAEpB;IAED,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,QAGzC;CAqFF;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,mBAAoB,YAAW,gBAAgB;;IAC1D,QAAQ,CAAC,EAAE,0BAA0B;IAOrC,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAe/C;IAED,GAAG,CAAC,CAAC,SAAS,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAclD;IAED,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CASnB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ1B;CAoCF;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,WAAW,EAAE,MAAM,IAAI,CAAC;CACzB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IASpB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAYrE;IAED,OAAO,SAeN;CACF;AAED;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AAEtC;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA2Ef;AA8BD,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAoCzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;IAQ3B,YAAY,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAG/C;IAED,WAAW,CAAC,UAAU,EAAE,mBAAmB,QAgB1C;IAsCD,IAAI,SAKH;CACF"}
+{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,IAAI,gBAAgB,EAAC,MAAM,aAAa,CAAC;AAC7D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAG/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,QAAQ,CAAU,CAAC;AAChE,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,SAAS,CAAU,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;IAWzB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAyC7C;IAEK,QAAQ,kBAKb;IAuBD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,QAsB7D;IAKD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAwBhE;IAED,YAAY,IAAI,MAAM,EAAE,CAEvB;IAEK,eAAe,kBAEpB;IAED,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,QAGzC;CAqFF;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,mBAAoB,YAAW,gBAAgB;;IAC1D,QAAQ,CAAC,EAAE,0BAA0B;IAOrC,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAe/C;IAED,GAAG,CAAC,CAAC,SAAS,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAclD;IAED,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CASnB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ1B;CAoCF;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,WAAW,EAAE,MAAM,IAAI,CAAC;CACzB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IASpB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAYrE;IAED,OAAO,SAeN;CACF;AAED;;;GAGG;AACH,eAAO,MAAM,SAAS,aAAa,CAAC;AAEpC;;;;;;;;GAQG;AACH,qBAAa,QAAS,SAAQ,gBAAgB;;IAQ5C,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAW9D;CA4BF;AAED;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AAEtC;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA2Ef;AA8BD,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAoCzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;IAQ3B,YAAY,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAG/C;IAED,WAAW,CAAC,UAAU,EAAE,mBAAmB,QAgB1C;IAsCD,IAAI,SAKH;CACF"}

package/out/zero-cache/src/services/life-cycle.js

@@ -1,12 +1,13 @@
+import { resolver } from "../../../shared/src/resolver.js";
 import { AbortError } from "../../../shared/src/abort-error.js";
 import { singleProcessMode } from "../types/processes.js";
-import { LONG_DURATION_HISTOGRAM_BOUNDARIES_S, getOrCreateHistogram } from "../observability/metrics.js";
 import { ConfigurationError } from "../types/configuration-error.js";
 import { logLastChanceSQLiteCorruptionDiagnostics } from "../db/sqlite-corruption.js";
+import { LONG_DURATION_HISTOGRAM_BOUNDARIES_S, getOrCreateHistogram } from "../observability/metrics.js";
 import { orTimeout } from "../types/timeout.js";
 import { RunningState } from "./running-state.js";
-import { resolver } from "@rocicorp/resolver";
 import { pid } from "node:process";
+import { EventEmitter } from "node:events";
 //#region ../zero-cache/src/services/life-cycle.ts
 var GRACEFUL_SHUTDOWN = ["SIGTERM", "SIGINT"];
 var FORCEFUL_SHUTDOWN = ["SIGQUIT", "SIGABRT"];
@@ -261,6 +262,57 @@ var SelfDrain = class {
 	}
 };
 /**
+* The event that a {@link PreDrain} emits immediately upon the first graceful
+* shutdown signal, with that signal as its argument, before the delay.
+*/
+var PRE_DRAIN = "preDrain";
+/**
+* Delays graceful shutdown signals (`SIGTERM`, `SIGINT`), e.g. so that load
+* balancers can observe a failed readiness check before the server begins
+* to drain. The first graceful signal is announced immediately with a
+* {@link PRE_DRAIN} event, and is re-emitted to listeners of the signal, such
+* as those of the {@link ProcessManager} and {@link runUntilKilled}, once the
+* delay has elapsed. Repeated graceful signals are dropped. All other events,
+* including forceful signals, are forwarded immediately.
+*/
+var PreDrain = class extends EventEmitter {
+	#lc;
+	#proc;
+	#delayMs;
+	#forwarded = /* @__PURE__ */ new Set(["newListener", PRE_DRAIN]);
+	#signaled = false;
+	#timer;
+	constructor(lc, proc, delayMs) {
+		super();
+		this.#lc = lc.withContext("component", "pre-drain");
+		this.#proc = proc;
+		this.#delayMs = delayMs;
+		for (const signal of GRACEFUL_SHUTDOWN) {
+			this.#forwarded.add(signal);
+			proc.on(signal, () => this.#onGracefulSignal(signal));
+		}
+		this.on("newListener", (event) => this.#forward(event));
+	}
+	#forward(event) {
+		if (this.#forwarded.has(event)) return;
+		this.#forwarded.add(event);
+		this.#proc.on(event, (...args) => {
+			if (event === "disconnect" || FORCEFUL_SHUTDOWN.includes(event)) clearTimeout(this.#timer);
+			this.emit(event, ...args);
+		});
+	}
+	#onGracefulSignal(signal) {
+		if (this.#signaled) {
+			this.#lc.info?.(`already shutting down. ignoring ${signal}`);
+			return;
+		}
+		this.#signaled = true;
+		this.#lc.info?.(`received ${signal}. draining in ${this.#delayMs} ms`);
+		this.emit(PRE_DRAIN, signal);
+		this.#timer = setTimeout(() => this.emit(signal, signal), this.#delayMs);
+	}
+};
+/**
 * The maximum time that {@link runUntilKilled} waits for its services to stop
 * (i.e. for their `run()` to settle) after stopping them.
 */
@@ -397,6 +449,6 @@ var HeartbeatMonitor = class {
 	}
 };
 //#endregion
-export { HeartbeatMonitor, ProcessManager, SelfDrain, SupportingProcesses, exitAfter, recordStartupDurationMs, runUntilKilled };
+export { FORCEFUL_SHUTDOWN, GRACEFUL_SHUTDOWN, HeartbeatMonitor, PRE_DRAIN, PreDrain, ProcessManager, STOP_TIMEOUT_MS, SelfDrain, SupportingProcesses, exitAfter, recordStartupDurationMs, runUntilKilled };
 
 //# sourceMappingURL=life-cycle.js.map