@rocicorp/zero 1.12.0-canary.3 → 1.12.0-canary.5

package/out/zero-cache/src/services/change-streamer/subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#wsBatched","#backlogBackpressure","#backlogFullWaiters","#onAck","#backfills","#onAligned","#onBackfillIgnored","#lagReportingDisabled","#caughtUp","#watermark","#acked","#backlog","#aligned","#bufferedBacklogBytes","#pushBacklog","#maybeWaitForBacklogSpace","#sendChange","#initialized","#trackBackfills","#sendPreSerializedDownstream","#sendDownstream","#initialize","#backlogDrain","#drainBacklog","#sendStringifiedDownstream","#advanceAck","#checkCaughtUp","#caughtUpWatermark","#pending","#pendingBytes","#processed","#backlogCount","#samples","#totalBufferedBytes","#missedLastTimeout","#laggingSinceMs","#backlogBytes","#resolveBacklogFullWaiters","#backlogInFlightBytes","#lowWaterBytes","#waiters"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n  wsBatched?: boolean | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n\n  /**\n   * The subscriber's pending backfills (protocol v8+), which are tracked\n   * over the changes sent to the subscriber.\n   */\n  backfills?: BackfillState | undefined;\n\n  /**\n   * Called once the subscriber has been sent all changes up to the head of\n   * the stream, i.e. its catchup and backlog have been flushed. From this\n   * point on, the subscriber's tracked {@link backfills} reflect the same\n   * position in the stream as that of the change-streamer.\n   */\n  onAligned?: ((subscriber: Subscriber) => void) | undefined;\n\n  /**\n   * Called when a `backfill` or `backfill-completed` message is ignored for\n   * some of the subscriber's pending backfill columns, after the subscriber\n   * is aligned.\n   */\n  onBackfillIgnored?: ((columns: string[]) => void) | undefined;\n\n  /**\n   * Resolves to whether lag reporting is disabled upstream. If so, a\n   * {@link LAG_REPORTING_DISABLED} status is sent once the subscriber is\n   * aligned, as a stand-in for the lag reports that a subscriber would\n   * otherwise use to determine that it is caught up (e.g. to advertise\n   * readiness to serve traffic).\n   */\n  lagReportingDisabled?: Promise<boolean> | undefined;\n\n  /**\n   * Whether the subscriber switches over to another replication-manager\n   * when this one drains (see `DrainTracker`).\n   */\n  drainable?: boolean | undefined;\n};\n\n/**\n * A status message queued in the backlog, so that it is sent downstream in\n * the order in which it was received relative to (backlogged) changes.\n */\ntype QueuedStatus = {readonly status: string};\n\ntype BacklogEntry = WatermarkedChange | QueuedStatus;\n\nfunction isQueuedStatus(entry: BacklogEntry): entry is QueuedStatus {\n  return !Array.isArray(entry);\n}\n\nfunction entryBytes(entry: BacklogEntry): number {\n  return isQueuedStatus(entry) ? entry.status.length : entry[2].length;\n}\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly drainable: boolean;\n  readonly #downstream: Subscription<string | PreSerializedBatch>;\n  readonly #wsBatched: boolean;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<BacklogEntry> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n  readonly #backfills: BackfillState | undefined;\n  readonly #onAligned: ((subscriber: Subscriber) => void) | undefined;\n  readonly #onBackfillIgnored: ((columns: string[]) => void) | undefined;\n  readonly #lagReportingDisabled: Promise<boolean> | undefined;\n  #aligned = false;\n  // The watermark to be acked for the subscriber to be caught up, set when\n  // it is aligned.\n  #caughtUpWatermark: string | undefined;\n  readonly #caughtUp = resolver<void>();\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string | PreSerializedBatch>,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.drainable = options.drainable ?? false;\n    this.#downstream = downstream;\n    this.#wsBatched = options.wsBatched ?? false;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n    this.#backfills = options.backfills;\n    this.#onAligned = options.onAligned;\n    this.#onBackfillIgnored = options.onBackfillIgnored;\n    this.#lagReportingDisabled = options.lagReportingDisabled;\n  }\n\n  /**\n   * The subscriber's tracked pending backfills, or `undefined` if the\n   * subscriber does not report them (i.e. protocol < v8).\n   */\n  get backfills(): BackfillState | undefined {\n    return this.#backfills;\n  }\n\n  /**\n   * Whether the subscriber has been sent all changes up to the head of the\n   * stream (see {@link SubscriberOptions.onAligned}).\n   */\n  get aligned(): boolean {\n    return this.#aligned;\n  }\n\n  /**\n   * Resolves once the subscriber has caught up, i.e. once it has acked\n   * (durably applied) all changes up to the head of the stream at the time\n   * it was {@link aligned}. Never resolves if the subscriber is closed first.\n   */\n  get caughtUp(): Promise<void> {\n    return this.#caughtUp.promise;\n  }\n\n  #trackBackfills(change: WatermarkedChange) {\n    if (this.#backfills) {\n      const ignored = this.#backfills.applySerialized(change);\n      if (ignored.length && this.#aligned) {\n        this.#onBackfillIgnored?.(ignored);\n      }\n    }\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  sendBatch(\n    changes: readonly WatermarkedChange[],\n    preSerialized?: PreSerializedBatch | undefined,\n  ): Promise<void> {\n    if (changes.length === 0) {\n      return promiseVoid;\n    }\n\n    if (\n      this.#wsBatched &&\n      preSerialized &&\n      !this.#backlog &&\n      this.#initialized &&\n      changes[0][0] > this.#watermark &&\n      (this.#protocolVersion >= 5 ||\n        changes.every(c => this.supportsMessage(c[1])))\n    ) {\n      let commitWatermark: string | undefined;\n      for (let i = changes.length - 1; i >= 0; i--) {\n        if (changes[i][1] === 'commit') {\n          commitWatermark = changes[i][0];\n          break;\n        }\n      }\n      if (commitWatermark) {\n        this.#watermark = commitWatermark;\n      }\n      for (const change of changes) {\n        this.#trackBackfills(change);\n      }\n      return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n    }\n\n    const promises: Promise<void>[] = [];\n    for (const change of changes) {\n      const p = this.send(change);\n      if (p !== promiseVoid) {\n        promises.push(p);\n      }\n    }\n    if (promises.length === 0) {\n      return promiseVoid;\n    }\n    if (promises.length === 1) {\n      return promises[0];\n    }\n    return Promise.all(promises).then(() => {});\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      // The initial status precedes the catchup, so it bypasses the backlog.\n      void this.#sendDownstream(['status', {tag: 'status'}]);\n    }\n  }\n\n  /**\n   * Sends a status message (e.g. a lag report) downstream. While the\n   * subscriber is catching up, the status is queued in the backlog behind\n   * the changes that preceded it in the replication stream, so that the\n   * subscriber processes it in stream order. This is necessary for lag\n   * reports to measure the lag of the subscriber, including its catchup.\n   *\n   * Status messages received before the subscriber is initialized are\n   * dropped.\n   */\n  sendStatus(status: Status) {\n    if (!this.#initialized) {\n      return;\n    }\n    const downstream: Downstream = ['status', status];\n    if (this.#backlog) {\n      this.#pushBacklog({status: BigIntJSON.stringify(downstream)});\n      return;\n    }\n    void this.#sendDownstream(downstream);\n  }\n\n  /**\n   * Sends a control status (e.g. `draining` or `restore`) downstream\n   * immediately, regardless of whether the subscriber is initialized or\n   * catching up. Unlike {@link sendStatus}, such statuses are not tied to a\n   * position in the replication stream.\n   */\n  sendControlStatus(status: Status) {\n    void this.#sendDownstream(['status', status]);\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    this.#trackBackfills(change);\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      this.#advanceAck(watermark);\n    }\n  }\n\n  #advanceAck(watermark: string) {\n    // Sends can complete out of order (e.g. the bounded window in\n    // #drainBacklog), so the ack only advances monotonically, and listeners\n    // are only notified when it does.\n    const acked = max(this.#acked, watermark);\n    if (acked !== this.#acked) {\n      this.#acked = acked;\n      this.#onAck?.(acked);\n      this.#checkCaughtUp();\n    }\n  }\n\n  #checkCaughtUp() {\n    if (\n      this.#caughtUpWatermark !== undefined &&\n      this.#acked >= this.#caughtUpWatermark\n    ) {\n      this.#caughtUpWatermark = undefined;\n      this.#caughtUp.resolve();\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  async #sendPreSerializedDownstream(\n    batch: PreSerializedBatch,\n    commitWatermark?: string | undefined,\n  ): Promise<void> {\n    const size = batch.byteLength;\n    this.#pending += batch.changes.length;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(batch);\n    try {\n      const outcome = await result;\n      if (commitWatermark && outcome === 'consumed') {\n        this.#advanceAck(commitWatermark);\n      }\n    } finally {\n      this.#pending -= batch.changes.length;\n      this.#pendingBytes -= size;\n      this.#processed += batch.changes.length;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(\n    error?: ErrorType,\n    message?: string,\n    details?: {replicaVersion?: string | undefined},\n  ) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream([\n        'error',\n        {type: error, message, ...details},\n      ]).finally(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(entry: BacklogEntry) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(entry);\n    this.#backlogBytes += entryBytes(entry);\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const entry = this.#backlog?.shift();\n        if (!entry) {\n          const closed = this.#backlog === null;\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          if (!closed && !this.#aligned) {\n            // All changes up to the head have been handed to #sendChange()\n            // (and thus tracked), and subsequent changes are sent directly.\n            this.#aligned = true;\n            this.#onAligned?.(this);\n            this.#caughtUpWatermark = this.#watermark;\n            this.#checkCaughtUp();\n            void this.#lagReportingDisabled?.then(disabled => {\n              if (disabled) {\n                this.sendStatus(LAG_REPORTING_DISABLED);\n              }\n            });\n          }\n          break;\n        }\n\n        const bytes = entryBytes(entry);\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const send = isQueuedStatus(entry)\n          ? this.#sendStringifiedDownstream(entry.status).then(() => {})\n          : this.#sendChange(entry);\n        const promise = send.finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n\n/**\n * Sent to an aligned subscriber when lag reporting is disabled upstream,\n * signaling that the subscriber is caught up. This is a lag report with a\n * `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured\n * (see `isLagReportingDisabledSignal()` in the replicator's recorder).\n */\nexport const LAG_REPORTING_DISABLED: Status = {\n  tag: 'status',\n  lagReport: {nextSendTimeMs: 0},\n};\n"],"mappings":";;;;;;;;AAsBA,IAAM,mCAAmC,KAAK,OAAO;AACrD,IAAM,kCAAkC;AA2DxC,SAAS,eAAe,OAA4C;CAClE,OAAO,CAAC,MAAM,QAAQ,KAAK;AAC7B;AAEA,SAAS,WAAW,OAA6B;CAC/C,OAAO,eAAe,KAAK,IAAI,MAAM,OAAO,SAAS,MAAM,GAAG;AAChE;;;;;;;;AAuBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CACA;CACA;CACA;CACA;CACA,WAAW;CAGX;CACA,YAAqB,SAAe;CAEpC,YACE,iBACA,IACA,MACA,WACA,YACA,UAA6B,CAAC,GAC9B;EACA,KAAKA,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAK,YAAY,QAAQ,aAAa;EACtC,KAAKC,cAAc;EACnB,KAAKC,aAAa,QAAQ,aAAa;EACvC,KAAKS,aAAa;EAClB,KAAKC,SAAS;EACd,KAAKC,WAAW,IAAI,WAAW;EAC/B,KAAKV,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAKE,SAAS,QAAQ;EACtB,KAAKC,aAAa,QAAQ;EAC1B,KAAKC,aAAa,QAAQ;EAC1B,KAAKC,qBAAqB,QAAQ;EAClC,KAAKC,wBAAwB,QAAQ;CACvC;;;;;CAMA,IAAI,YAAuC;EACzC,OAAO,KAAKH;CACd;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAKQ;CACd;;;;;;CAOA,IAAI,WAA0B;EAC5B,OAAO,KAAKJ,UAAU;CACxB;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAKJ,YAAY;GACnB,MAAM,UAAU,KAAKA,WAAW,gBAAgB,MAAM;GACtD,IAAI,QAAQ,UAAU,KAAKQ,UACzB,KAAKN,qBAAqB,OAAO;EAErC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAKG;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAKC;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAKG,yBAAyB,KAAKZ,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAKU,aAAa,QAAQ,KAAKE,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAKX,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAKA,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAKO,YAAY;GAC/B,IAAI,KAAKE,UAAU;IAIjB,KAAKG,aAAa,MAAM;IACxB,OAAO,KAAKC,0BAA0B;GACxC;GACA,OAAO,KAAKC,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAKhB,cACL,iBACA,CAAC,KAAKW,YACN,KAAKM,gBACL,QAAQ,GAAG,KAAK,KAAKR,eACpB,KAAKX,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,GAAG,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,GAAG;IAC7B;GACF;GAEF,IAAI,iBACF,KAAKW,aAAa;GAEpB,KAAK,MAAM,UAAU,SACnB,KAAKS,gBAAgB,MAAM;GAE7B,OAAO,KAAKC,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,EAAE,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAKF,cAAc;GACtB,KAAKA,eAAe;GAEpB,KAAUG,gBAAgB,CAAC,UAAU,EAAC,KAAK,SAAQ,CAAC,CAAC;EACvD;CACF;;;;;;;;;;;CAYA,WAAW,QAAgB;EACzB,IAAI,CAAC,KAAKH,cACR;EAEF,MAAM,aAAyB,CAAC,UAAU,MAAM;EAChD,IAAI,KAAKN,UAAU;GACjB,KAAKG,aAAa,EAAC,QAAQ,WAAW,UAAU,UAAU,EAAC,CAAC;GAC5D;EACF;EACA,KAAUM,gBAAgB,UAAU;CACtC;;;;;;;CAQA,kBAAkB,QAAgB;EAChC,KAAUA,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAC9C;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAKC,YAAY;EACjB,MAAM,KAAKL,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAKK,YAAY;EACjB,IAAI,CAAC,KAAKV,UACR,OAAO,KAAKW,iBAAiB;EAE/B,IAAI,CAAC,KAAKA,eAAe;GAKvB,KAAKA,gBAAgB,KAAKC,cAAc;GACxC,KAAUD,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAKA;CACd;CAEA,MAAMN,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAKP,aAAa;EAEpB,KAAKS,gBAAgB,MAAM;EAC3B,MAAM,SAAS,MAAM,KAAKM,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YACjC,KAAKC,YAAY,SAAS;CAE9B;CAEA,YAAY,WAAmB;EAI7B,MAAM,QAAQ,IAAI,KAAKf,QAAQ,SAAS;EACxC,IAAI,UAAU,KAAKA,QAAQ;GACzB,KAAKA,SAAS;GACd,KAAKP,SAAS,KAAK;GACnB,KAAKuB,eAAe;EACtB;CACF;CAEA,iBAAiB;EACf,IACE,KAAKC,uBAAuB,KAAA,KAC5B,KAAKjB,UAAU,KAAKiB,oBACpB;GACA,KAAKA,qBAAqB,KAAA;GAC1B,KAAKnB,UAAU,QAAQ;EACzB;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAKgB,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAMA,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAKI;EACL,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK9B,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAK6B;GACL,KAAKC,iBAAiB;GACtB,KAAKC;EACP;CACF;CAEA,MAAMX,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAKS,YAAY,MAAM,QAAQ;EAC/B,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK9B,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YACjC,KAAK0B,YAAY,eAAe;EAEpC,UAAU;GACR,KAAKG,YAAY,MAAM,QAAQ;GAC/B,KAAKC,iBAAiB;GACtB,KAAKC,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAKF,WAAW,KAAKG;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAKD;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAKE,SAAS,UAAU,YAC7B,KAAKA,SAAS,MAAM;EAEtB,KAAKA,SAAS,KAAK;GAAC,WAAW,KAAKF;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAKE,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAKD;GACd,cAAc,KAAKlB;GACnB,oBAAoB,KAAKoB;GACzB,mBAAmB,KAAKC;EAC1B;EAEF,MAAM,OAAO,KAAKF,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAKA,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAKD;GACd,cAAc,KAAKlB;GACnB,oBAAoB,KAAKoB;GACzB,mBAAmB,KAAKC;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAKA,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAKC,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAKD,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAKC,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAKA,oBAAoB;EACzB,OAAO,MAAM,KAAKA;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAKrC,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MACE,OACA,SACA,SACA;EAGA,KAAKa,WAAW;EAChB,KAAKyB,gBAAgB;EACrB,KAAKnC,qBAAqB,WAAW;EAGrC,KAAKoC,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAUjB,gBAAgB,CACxB,SACA;GAAC,MAAM;GAAO;GAAS,GAAG;EAAO,CACnC,CAAC,EAAE,cAAc,KAAKrB,YAAY,OAAO,CAAC;OAE1C,KAAKA,YAAY,OAAO;CAE5B;CAEA,IAAIgC,gBAAgB;EAClB,OAAO,KAAKpB,UAAU,QAAQ;CAChC;CAEA,IAAIE,wBAAwB;EAI1B,OAAO,KAAKuB,gBAAgB,KAAKE;CACnC;CAEA,IAAIL,sBAAsB;EACxB,OAAO,KAAKpB,wBAAwB,KAAKgB;CAC3C;CAEA,aAAa,OAAqB;EAChC,OAAO,KAAKlB,UAAU,gDAAgD;EACtE,KAAKA,SAAS,KAAK,KAAK;EACxB,KAAKyB,iBAAiB,WAAW,KAAK;EACtC,IAAI,KAAK,aACP,KAAKC,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAKnC,mBAAmB;EAC5C,KAAKA,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAKD,qBAAqB,aAAa,KAAKY,qBAAqB;CAC1E;CAEA,MAAMU,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,QAAQ,KAAKZ,UAAU,MAAM;IACnC,IAAI,CAAC,OAAO;KACV,MAAM,SAAS,KAAKA,aAAa;KACjC,KAAKA,WAAW;KAChB,KAAKyB,gBAAgB;KACrB,KAAKnC,qBAAqB,uBACxB,KAAKY,qBACP;KACA,IAAI,CAAC,UAAU,CAAC,KAAKD,UAAU;MAG7B,KAAKA,WAAW;MAChB,KAAKP,aAAa,IAAI;MACtB,KAAKsB,qBAAqB,KAAKlB;MAC/B,KAAKiB,eAAe;MACpB,KAAUnB,uBAAuB,MAAK,aAAY;OAChD,IAAI,UACF,KAAK,WAAW,sBAAsB;MAE1C,CAAC;KACH;KACA;IACF;IAEA,MAAM,QAAQ,WAAW,KAAK;IAC9B,KAAK6B,iBAAiB;IACtB,KAAKE,yBAAyB;IAC9B,KAAKrC,qBAAqB,uBACxB,KAAKY,qBACP;IAQA,MAAM,WAHO,eAAe,KAAK,IAC7B,KAAKW,2BAA2B,MAAM,MAAM,EAAE,WAAW,CAAC,CAAC,IAC3D,KAAKR,YAAY,KAAK,GACL,cAAc;KACjC,KAAKsB,yBAAyB;KAC9B,KAAKrC,qBAAqB,uBACxB,KAAKY,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAKZ,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAKqB,gBAAgB;GACrB,KAAKrB,qBAAqB,uBACxB,KAAKY,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAK0B,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAKC,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAKA,SAAS,WAAW,KAAK,gBAAgB,KAAKD,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAKC,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF;;;;;;;AAQA,IAAa,yBAAiC;CAC5C,KAAK;CACL,WAAW,EAAC,gBAAgB,EAAC;AAC/B"}
+{"version":3,"file":"subscriber.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n  wsBatched?: boolean | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n\n  /**\n   * The subscriber's pending backfills (protocol v8+), which are tracked\n   * over the changes sent to the subscriber.\n   */\n  backfills?: BackfillState | undefined;\n\n  /**\n   * Called once the subscriber has been sent all changes up to the head of\n   * the stream, i.e. its catchup and backlog have been flushed. From this\n   * point on, the subscriber's tracked {@link backfills} reflect the same\n   * position in the stream as that of the change-streamer.\n   */\n  onAligned?: ((subscriber: Subscriber) => void) | undefined;\n\n  /**\n   * Called when a `backfill` or `backfill-completed` message is ignored for\n   * some of the subscriber's pending backfill columns, after the subscriber\n   * is aligned.\n   */\n  onBackfillIgnored?: ((columns: string[]) => void) | undefined;\n\n  /**\n   * Resolves to whether lag reporting is disabled upstream. If so, a\n   * {@link LAG_REPORTING_DISABLED} status is sent once the subscriber is\n   * aligned, as a stand-in for the lag reports that a subscriber would\n   * otherwise use to determine that it is caught up (e.g. to advertise\n   * readiness to serve traffic).\n   */\n  lagReportingDisabled?: Promise<boolean> | undefined;\n\n  /**\n   * Whether the subscriber switches over to another replication-manager\n   * when this one drains (see `DrainTracker`).\n   */\n  drainable?: boolean | undefined;\n};\n\n/**\n * A status message queued in the backlog, so that it is sent downstream in\n * the order in which it was received relative to (backlogged) changes.\n */\ntype QueuedStatus = {readonly status: string};\n\ntype BacklogEntry = WatermarkedChange | QueuedStatus;\n\nfunction isQueuedStatus(entry: BacklogEntry): entry is QueuedStatus {\n  return !Array.isArray(entry);\n}\n\nfunction entryBytes(entry: BacklogEntry): number {\n  return isQueuedStatus(entry) ? entry.status.length : entry[2].length;\n}\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly drainable: boolean;\n  readonly #downstream: Subscription<string | PreSerializedBatch>;\n  readonly #wsBatched: boolean;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<BacklogEntry> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n  readonly #backfills: BackfillState | undefined;\n  readonly #onAligned: ((subscriber: Subscriber) => void) | undefined;\n  readonly #onBackfillIgnored: ((columns: string[]) => void) | undefined;\n  readonly #lagReportingDisabled: Promise<boolean> | undefined;\n  #aligned = false;\n  // The watermark to be acked for the subscriber to be caught up, set when\n  // it is aligned.\n  #caughtUpWatermark: string | undefined;\n  readonly #caughtUp = resolver<void>();\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string | PreSerializedBatch>,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.drainable = options.drainable ?? false;\n    this.#downstream = downstream;\n    this.#wsBatched = options.wsBatched ?? false;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n    this.#backfills = options.backfills;\n    this.#onAligned = options.onAligned;\n    this.#onBackfillIgnored = options.onBackfillIgnored;\n    this.#lagReportingDisabled = options.lagReportingDisabled;\n  }\n\n  /**\n   * The subscriber's tracked pending backfills, or `undefined` if the\n   * subscriber does not report them (i.e. protocol < v8).\n   */\n  get backfills(): BackfillState | undefined {\n    return this.#backfills;\n  }\n\n  /**\n   * Whether the subscriber has been sent all changes up to the head of the\n   * stream (see {@link SubscriberOptions.onAligned}).\n   */\n  get aligned(): boolean {\n    return this.#aligned;\n  }\n\n  /**\n   * Resolves once the subscriber has caught up, i.e. once it has acked\n   * (durably applied) all changes up to the head of the stream at the time\n   * it was {@link aligned}. Never resolves if the subscriber is closed first.\n   */\n  get caughtUp(): Promise<void> {\n    return this.#caughtUp.promise;\n  }\n\n  #trackBackfills(change: WatermarkedChange) {\n    if (this.#backfills) {\n      const ignored = this.#backfills.applySerialized(change);\n      if (ignored.length && this.#aligned) {\n        this.#onBackfillIgnored?.(ignored);\n      }\n    }\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  sendBatch(\n    changes: readonly WatermarkedChange[],\n    preSerialized?: PreSerializedBatch | undefined,\n  ): Promise<void> {\n    if (changes.length === 0) {\n      return promiseVoid;\n    }\n\n    if (\n      this.#wsBatched &&\n      preSerialized &&\n      !this.#backlog &&\n      this.#initialized &&\n      changes[0][0] > this.#watermark &&\n      (this.#protocolVersion >= 5 ||\n        changes.every(c => this.supportsMessage(c[1])))\n    ) {\n      let commitWatermark: string | undefined;\n      for (let i = changes.length - 1; i >= 0; i--) {\n        if (changes[i][1] === 'commit') {\n          commitWatermark = changes[i][0];\n          break;\n        }\n      }\n      if (commitWatermark) {\n        this.#watermark = commitWatermark;\n      }\n      for (const change of changes) {\n        this.#trackBackfills(change);\n      }\n      return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n    }\n\n    const promises: Promise<void>[] = [];\n    for (const change of changes) {\n      const p = this.send(change);\n      if (p !== promiseVoid) {\n        promises.push(p);\n      }\n    }\n    if (promises.length === 0) {\n      return promiseVoid;\n    }\n    if (promises.length === 1) {\n      return promises[0];\n    }\n    return Promise.all(promises).then(() => {});\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      // The initial status precedes the catchup, so it bypasses the backlog.\n      void this.#sendDownstream(['status', {tag: 'status'}]);\n    }\n  }\n\n  /**\n   * Sends a status message (e.g. a lag report) downstream. While the\n   * subscriber is catching up, the status is queued in the backlog behind\n   * the changes that preceded it in the replication stream, so that the\n   * subscriber processes it in stream order. This is necessary for lag\n   * reports to measure the lag of the subscriber, including its catchup.\n   *\n   * Status messages received before the subscriber is initialized are\n   * dropped.\n   */\n  sendStatus(status: Status) {\n    if (!this.#initialized) {\n      return;\n    }\n    const downstream: Downstream = ['status', status];\n    if (this.#backlog) {\n      this.#pushBacklog({status: BigIntJSON.stringify(downstream)});\n      return;\n    }\n    void this.#sendDownstream(downstream);\n  }\n\n  /**\n   * Sends a control status (e.g. `draining` or `restore`) downstream\n   * immediately, regardless of whether the subscriber is initialized or\n   * catching up. Unlike {@link sendStatus}, such statuses are not tied to a\n   * position in the replication stream.\n   */\n  sendControlStatus(status: Status) {\n    void this.#sendDownstream(['status', status]);\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    this.#trackBackfills(change);\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      this.#advanceAck(watermark);\n    }\n  }\n\n  #advanceAck(watermark: string) {\n    // Sends can complete out of order (e.g. the bounded window in\n    // #drainBacklog), so the ack only advances monotonically, and listeners\n    // are only notified when it does.\n    const acked = max(this.#acked, watermark);\n    if (acked !== this.#acked) {\n      this.#acked = acked;\n      this.#onAck?.(acked);\n      this.#checkCaughtUp();\n    }\n  }\n\n  #checkCaughtUp() {\n    if (\n      this.#caughtUpWatermark !== undefined &&\n      this.#acked >= this.#caughtUpWatermark\n    ) {\n      this.#caughtUpWatermark = undefined;\n      this.#caughtUp.resolve();\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  async #sendPreSerializedDownstream(\n    batch: PreSerializedBatch,\n    commitWatermark?: string | undefined,\n  ): Promise<void> {\n    const size = batch.byteLength;\n    this.#pending += batch.changes.length;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(batch);\n    try {\n      const outcome = await result;\n      if (commitWatermark && outcome === 'consumed') {\n        this.#advanceAck(commitWatermark);\n      }\n    } finally {\n      this.#pending -= batch.changes.length;\n      this.#pendingBytes -= size;\n      this.#processed += batch.changes.length;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(\n    error?: ErrorType,\n    message?: string,\n    details?: {replicaVersion?: string | undefined},\n  ) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream([\n        'error',\n        {type: error, message, ...details},\n      ]).finally(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(entry: BacklogEntry) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(entry);\n    this.#backlogBytes += entryBytes(entry);\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const entry = this.#backlog?.shift();\n        if (!entry) {\n          const closed = this.#backlog === null;\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          if (!closed && !this.#aligned) {\n            // All changes up to the head have been handed to #sendChange()\n            // (and thus tracked), and subsequent changes are sent directly.\n            this.#aligned = true;\n            this.#onAligned?.(this);\n            this.#caughtUpWatermark = this.#watermark;\n            this.#checkCaughtUp();\n            void this.#lagReportingDisabled?.then(disabled => {\n              if (disabled) {\n                this.sendStatus(LAG_REPORTING_DISABLED);\n              }\n            });\n          }\n          break;\n        }\n\n        const bytes = entryBytes(entry);\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const send = isQueuedStatus(entry)\n          ? this.#sendStringifiedDownstream(entry.status).then(() => {})\n          : this.#sendChange(entry);\n        const promise = send.finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n\n/**\n * Sent to an aligned subscriber when lag reporting is disabled upstream,\n * signaling that the subscriber is caught up. This is a lag report with a\n * `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured\n * (see `isLagReportingDisabledSignal()` in the replicator's recorder).\n */\nexport const LAG_REPORTING_DISABLED: Status = {\n  tag: 'status',\n  lagReport: {nextSendTimeMs: 0},\n};\n"],"mappings":";;;;;;;;AAsBA,IAAM,mCAAmC;AACzC,IAAM,kCAAkC;AA2DxC,SAAS,eAAe,OAA4C;CAClE,OAAO,CAAC,MAAM,QAAQ,KAAK;AAC7B;AAEA,SAAS,WAAW,OAA6B;CAC/C,OAAO,eAAe,KAAK,IAAI,MAAM,OAAO,SAAS,MAAM,EAAE,CAAC;AAChE;;;;;;;;AAuBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CACA;CACA;CACA;CACA;CACA,WAAW;CAGX;CACA,YAAqB,SAAe;CAEpC,YACE,iBACA,IACA,MACA,WACA,YACA,UAA6B,CAAC,GAC9B;EACA,KAAK,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAK,YAAY,QAAQ,aAAa;EACtC,KAAK,cAAc;EACnB,KAAK,aAAa,QAAQ,aAAa;EACvC,KAAK,aAAa;EAClB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,WAAW;EAC/B,KAAK,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAK,SAAS,QAAQ;EACtB,KAAK,aAAa,QAAQ;EAC1B,KAAK,aAAa,QAAQ;EAC1B,KAAK,qBAAqB,QAAQ;EAClC,KAAK,wBAAwB,QAAQ;CACvC;;;;;CAMA,IAAI,YAAuC;EACzC,OAAO,KAAK;CACd;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAK;CACd;;;;;;CAOA,IAAI,WAA0B;EAC5B,OAAO,KAAK,UAAU;CACxB;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAK,YAAY;GACnB,MAAM,UAAU,KAAK,WAAW,gBAAgB,MAAM;GACtD,IAAI,QAAQ,UAAU,KAAK,UACzB,KAAK,qBAAqB,OAAO;EAErC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAK;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAK;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAK,yBAAyB,KAAK,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAK,aAAa,QAAQ,KAAK,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAK,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAK,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAK,YAAY;GAC/B,IAAI,KAAK,UAAU;IAIjB,KAAK,aAAa,MAAM;IACxB,OAAO,KAAK,0BAA0B;GACxC;GACA,OAAO,KAAK,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAK,cACL,iBACA,CAAC,KAAK,YACN,KAAK,gBACL,QAAQ,EAAE,CAAC,KAAK,KAAK,eACpB,KAAK,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,EAAE,CAAC,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,EAAE,CAAC;IAC7B;GACF;GAEF,IAAI,iBACF,KAAK,aAAa;GAEpB,KAAK,MAAM,UAAU,SACnB,KAAK,gBAAgB,MAAM;GAE7B,OAAO,KAAK,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAK,cAAc;GACtB,KAAK,eAAe;GAEpB,KAAU,gBAAgB,CAAC,UAAU,EAAC,KAAK,SAAQ,CAAC,CAAC;EACvD;CACF;;;;;;;;;;;CAYA,WAAW,QAAgB;EACzB,IAAI,CAAC,KAAK,cACR;EAEF,MAAM,aAAyB,CAAC,UAAU,MAAM;EAChD,IAAI,KAAK,UAAU;GACjB,KAAK,aAAa,EAAC,QAAQ,WAAW,UAAU,UAAU,EAAC,CAAC;GAC5D;EACF;EACA,KAAU,gBAAgB,UAAU;CACtC;;;;;;;CAQA,kBAAkB,QAAgB;EAChC,KAAU,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAC9C;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAK,YAAY;EACjB,MAAM,KAAK,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAK,YAAY;EACjB,IAAI,CAAC,KAAK,UACR,OAAO,KAAK,iBAAiB;EAE/B,IAAI,CAAC,KAAK,eAAe;GAKvB,KAAK,gBAAgB,KAAK,cAAc;GACxC,KAAU,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAK;CACd;CAEA,MAAM,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAK,aAAa;EAEpB,KAAK,gBAAgB,MAAM;EAC3B,MAAM,SAAS,MAAM,KAAK,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YACjC,KAAK,YAAY,SAAS;CAE9B;CAEA,YAAY,WAAmB;EAI7B,MAAM,QAAQ,IAAI,KAAK,QAAQ,SAAS;EACxC,IAAI,UAAU,KAAK,QAAQ;GACzB,KAAK,SAAS;GACd,KAAK,SAAS,KAAK;GACnB,KAAK,eAAe;EACtB;CACF;CAEA,iBAAiB;EACf,IACE,KAAK,uBAAuB,KAAA,KAC5B,KAAK,UAAU,KAAK,oBACpB;GACA,KAAK,qBAAqB,KAAA;GAC1B,KAAK,UAAU,QAAQ;EACzB;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAK,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAM,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAK;EACL,KAAK,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAK;GACL,KAAK,iBAAiB;GACtB,KAAK;EACP;CACF;CAEA,MAAM,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAK,YAAY,MAAM,QAAQ;EAC/B,KAAK,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YACjC,KAAK,YAAY,eAAe;EAEpC,UAAU;GACR,KAAK,YAAY,MAAM,QAAQ;GAC/B,KAAK,iBAAiB;GACtB,KAAK,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAK,WAAW,KAAK;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAK;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAK,SAAS,UAAU,YAC7B,KAAK,SAAS,MAAM;EAEtB,KAAK,SAAS,KAAK;GAAC,WAAW,KAAK;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAK,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAK;GACd,cAAc,KAAK;GACnB,oBAAoB,KAAK;GACzB,mBAAmB,KAAK;EAC1B;EAEF,MAAM,OAAO,KAAK,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAK,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAK;GACd,cAAc,KAAK;GACnB,oBAAoB,KAAK;GACzB,mBAAmB,KAAK;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAK,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAK,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAK,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAK,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAK,oBAAoB;EACzB,OAAO,MAAM,KAAK;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAK,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MACE,OACA,SACA,SACA;EAGA,KAAK,WAAW;EAChB,KAAK,gBAAgB;EACrB,KAAK,qBAAqB,WAAW;EAGrC,KAAK,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAU,gBAAgB,CACxB,SACA;GAAC,MAAM;GAAO;GAAS,GAAG;EAAO,CACnC,CAAC,CAAC,CAAC,cAAc,KAAK,YAAY,OAAO,CAAC;OAE1C,KAAK,YAAY,OAAO;CAE5B;CAEA,IAAI,gBAAgB;EAClB,OAAO,KAAK,UAAU,QAAQ;CAChC;CAEA,IAAI,wBAAwB;EAI1B,OAAO,KAAK,gBAAgB,KAAK;CACnC;CAEA,IAAI,sBAAsB;EACxB,OAAO,KAAK,wBAAwB,KAAK;CAC3C;CAEA,aAAa,OAAqB;EAChC,OAAO,KAAK,UAAU,gDAAgD;EACtE,KAAK,SAAS,KAAK,KAAK;EACxB,KAAK,iBAAiB,WAAW,KAAK;EACtC,IAAI,KAAK,aACP,KAAK,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAK,mBAAmB;EAC5C,KAAK,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAK,qBAAqB,aAAa,KAAK,qBAAqB;CAC1E;CAEA,MAAM,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,QAAQ,KAAK,UAAU,MAAM;IACnC,IAAI,CAAC,OAAO;KACV,MAAM,SAAS,KAAK,aAAa;KACjC,KAAK,WAAW;KAChB,KAAK,gBAAgB;KACrB,KAAK,qBAAqB,uBACxB,KAAK,qBACP;KACA,IAAI,CAAC,UAAU,CAAC,KAAK,UAAU;MAG7B,KAAK,WAAW;MAChB,KAAK,aAAa,IAAI;MACtB,KAAK,qBAAqB,KAAK;MAC/B,KAAK,eAAe;MACpB,KAAU,uBAAuB,MAAK,aAAY;OAChD,IAAI,UACF,KAAK,WAAW,sBAAsB;MAE1C,CAAC;KACH;KACA;IACF;IAEA,MAAM,QAAQ,WAAW,KAAK;IAC9B,KAAK,iBAAiB;IACtB,KAAK,yBAAyB;IAC9B,KAAK,qBAAqB,uBACxB,KAAK,qBACP;IAQA,MAAM,WAHO,eAAe,KAAK,IAC7B,KAAK,2BAA2B,MAAM,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC,IAC3D,KAAK,YAAY,KAAK,EAAA,CACL,cAAc;KACjC,KAAK,yBAAyB;KAC9B,KAAK,qBAAqB,uBACxB,KAAK,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAK,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAK,gBAAgB;GACrB,KAAK,qBAAqB,uBACxB,KAAK,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAK,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAK,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAK,SAAS,WAAW,KAAK,gBAAgB,KAAK,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAK,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF;;;;;;;AAQA,IAAa,yBAAiC;CAC5C,KAAK;CACL,WAAW,EAAC,gBAAgB,EAAC;AAC/B"}

package/out/zero-cache/src/services/change-streamer/upstream-acker.js

@@ -69,7 +69,6 @@ var UpstreamAcker = class {
 			case "commit":
 				this.#lastTx = max(this.#lastTx, downstream[2].watermark);
 				this.#maybeAck();
-				break;
 		}
 	}
 	trackPgChangeLog(committedWatermark) {

package/out/zero-cache/src/services/change-streamer/upstream-acker.js.map

@@ -1 +1 @@
-{"version":3,"file":"upstream-acker.js","names":["#trackPgChangeLog","#trackBackup","#upstream","#lastTx","#lastStatus","#lastAck","#pgChangeLogWatermark","#maybeAck","#backupWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/upstream-acker.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport {max, min} from '../../types/lexi-version.ts';\nimport type {Sink} from '../../types/streams.ts';\nimport type {\n  ChangeSourceUpstream,\n  ChangeStreamMessage,\n} from '../change-source/protocol/current.ts';\n\ntype Opts = {\n  trackPgChangeLog: boolean;\n  trackBackup: boolean;\n};\n\n/**\n * Tracks the progress (watermark) of multiple streams:\n * - downstream transactions and status messages (the latter\n *   indicating LSNs that are not relevant to the publication\n *   but nevertheless need to be ACKed)\n * - PG change-log commits\n * - backup watermarks\n *\n * and sends upstream ACKs accordingly. When both the PG change-log\n * and backup watermarks are being considered for upstream ACKs\n * (i.e. RMv1.5), only watermarks that have been reached by both\n * stores are acked.\n *\n * The UpstreamAcker also takes into account that an upstream\n * connection can be disconnected and {@link reset()}. In this case,\n * the progress of the persistent stores is retained and the acks\n * are resent on the new upstream connection as necessary.\n */\nexport class UpstreamAcker {\n  readonly #trackPgChangeLog: boolean;\n  readonly #trackBackup: boolean;\n\n  #pgChangeLogWatermark = '';\n  #backupWatermark = '';\n\n  #upstream: Sink<ChangeSourceUpstream> | undefined;\n  #lastTx = '';\n  #lastStatus = '';\n  #lastAck = '';\n\n  constructor({trackPgChangeLog, trackBackup}: Opts) {\n    assert(\n      trackPgChangeLog || trackBackup,\n      `At least one of trackPgChangeLog or trackBackup must be true`,\n    );\n    this.#trackPgChangeLog = trackPgChangeLog;\n    this.#trackBackup = trackBackup;\n  }\n\n  /**\n   * Starts tracking a new upstream connection, which resumes the change stream\n   * after `resumeWatermark`.\n   *\n   * Everything at or before `resumeWatermark` was committed on an earlier\n   * connection, so it counts as committed on this one: a status watermark past\n   * it waits for the tracked stores to reach it, as it would for a commit on\n   * this connection. Otherwise the first keepalive of a stream that resumes\n   * ahead of the stores -- a change log ahead of its backup, which is the\n   * normal state -- would move the replication slot past transactions that the\n   * backup has not persisted, and a task restored from the backup would then\n   * resume below the slot, which the upstream moves forward without a word.\n   *\n   * When the PG change log is tracked, the stream resumes from its head, so\n   * `resumeWatermark` has been persisted in it (by this or another\n   * change-streamer). Without this, status messages would not be acked on an\n   * idle upstream until a commit is stored.\n   */\n  reset(upstream: Sink<ChangeSourceUpstream>, resumeWatermark: string) {\n    this.#upstream = upstream;\n    this.#lastTx = resumeWatermark;\n    this.#lastStatus = '';\n    this.#lastAck = '';\n    if (this.#trackPgChangeLog) {\n      this.#pgChangeLogWatermark = max(\n        resumeWatermark,\n        this.#pgChangeLogWatermark,\n      );\n    }\n    // Re-ack what the stores have persisted on the new connection.\n    this.#maybeAck();\n  }\n\n  trackDownstream(downstream: ChangeStreamMessage) {\n    const [tag, msg] = downstream;\n    switch (tag) {\n      case 'status':\n        if (msg.ack) {\n          this.#lastStatus = downstream[2].watermark;\n        }\n        this.#maybeAck();\n        break;\n      case 'commit':\n        // A commit replayed from below the resume watermark does not lower it.\n        this.#lastTx = max(this.#lastTx, downstream[2].watermark);\n        this.#maybeAck();\n        break;\n    }\n  }\n\n  trackPgChangeLog(committedWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#pgChangeLogWatermark = max(\n      committedWatermark,\n      this.#pgChangeLogWatermark,\n    );\n    this.#maybeAck();\n  }\n\n  trackBackup(backedUpWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#backupWatermark = max(backedUpWatermark, this.#backupWatermark);\n    this.#maybeAck();\n  }\n\n  #maybeAck() {\n    const currentWatermark = !this.#trackBackup\n      ? this.#pgChangeLogWatermark\n      : !this.#trackPgChangeLog\n        ? this.#backupWatermark\n        : min(this.#pgChangeLogWatermark, this.#backupWatermark);\n    if (currentWatermark > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {tag: 'commit'},\n        {watermark: currentWatermark},\n      ]);\n      this.#lastAck = currentWatermark;\n    }\n    // If all committed transactions have been acked, ack any outstanding\n    // status LSN's thereafter (i.e. non-publication upstream changes).\n    if (this.#lastAck >= this.#lastTx && this.#lastStatus > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {ack: true},\n        {watermark: this.#lastStatus},\n      ]);\n      this.#lastAck = this.#lastStatus;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+BA,IAAa,gBAAb,MAA2B;CACzB;CACA;CAEA,wBAAwB;CACxB,mBAAmB;CAEnB;CACA,UAAU;CACV,cAAc;CACd,WAAW;CAEX,YAAY,EAAC,kBAAkB,eAAoB;EACjD,OACE,oBAAoB,aACpB,8DACF;EACA,KAAKA,oBAAoB;EACzB,KAAKC,eAAe;CACtB;;;;;;;;;;;;;;;;;;;CAoBA,MAAM,UAAsC,iBAAyB;EACnE,KAAKC,YAAY;EACjB,KAAKC,UAAU;EACf,KAAKC,cAAc;EACnB,KAAKC,WAAW;EAChB,IAAI,KAAKL,mBACP,KAAKM,wBAAwB,IAC3B,iBACA,KAAKA,qBACP;EAGF,KAAKC,UAAU;CACjB;CAEA,gBAAgB,YAAiC;EAC/C,MAAM,CAAC,KAAK,OAAO;EACnB,QAAQ,KAAR;GACE,KAAK;IACH,IAAI,IAAI,KACN,KAAKH,cAAc,WAAW,GAAG;IAEnC,KAAKG,UAAU;IACf;GACF,KAAK;IAEH,KAAKJ,UAAU,IAAI,KAAKA,SAAS,WAAW,GAAG,SAAS;IACxD,KAAKI,UAAU;IACf;EACJ;CACF;CAEA,iBAAiB,oBAA4B;EAE3C,KAAKD,wBAAwB,IAC3B,oBACA,KAAKA,qBACP;EACA,KAAKC,UAAU;CACjB;CAEA,YAAY,mBAA2B;EAErC,KAAKC,mBAAmB,IAAI,mBAAmB,KAAKA,gBAAgB;EACpE,KAAKD,UAAU;CACjB;CAEA,YAAY;EACV,MAAM,mBAAmB,CAAC,KAAKN,eAC3B,KAAKK,wBACL,CAAC,KAAKN,oBACJ,KAAKQ,mBACL,IAAI,KAAKF,uBAAuB,KAAKE,gBAAgB;EAC3D,IAAI,mBAAmB,KAAKH,UAAU;GACpC,KAAKH,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,SAAQ;IACd,EAAC,WAAW,iBAAgB;GAC9B,CAAC;GACD,KAAKG,WAAW;EAClB;EAGA,IAAI,KAAKA,YAAY,KAAKF,WAAW,KAAKC,cAAc,KAAKC,UAAU;GACrE,KAAKH,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,KAAI;IACV,EAAC,WAAW,KAAKE,YAAW;GAC9B,CAAC;GACD,KAAKC,WAAW,KAAKD;EACvB;CACF;AACF"}
+{"version":3,"file":"upstream-acker.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/upstream-acker.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport {max, min} from '../../types/lexi-version.ts';\nimport type {Sink} from '../../types/streams.ts';\nimport type {\n  ChangeSourceUpstream,\n  ChangeStreamMessage,\n} from '../change-source/protocol/current.ts';\n\ntype Opts = {\n  trackPgChangeLog: boolean;\n  trackBackup: boolean;\n};\n\n/**\n * Tracks the progress (watermark) of multiple streams:\n * - downstream transactions and status messages (the latter\n *   indicating LSNs that are not relevant to the publication\n *   but nevertheless need to be ACKed)\n * - PG change-log commits\n * - backup watermarks\n *\n * and sends upstream ACKs accordingly. When both the PG change-log\n * and backup watermarks are being considered for upstream ACKs\n * (i.e. RMv1.5), only watermarks that have been reached by both\n * stores are acked.\n *\n * The UpstreamAcker also takes into account that an upstream\n * connection can be disconnected and {@link reset()}. In this case,\n * the progress of the persistent stores is retained and the acks\n * are resent on the new upstream connection as necessary.\n */\nexport class UpstreamAcker {\n  readonly #trackPgChangeLog: boolean;\n  readonly #trackBackup: boolean;\n\n  #pgChangeLogWatermark = '';\n  #backupWatermark = '';\n\n  #upstream: Sink<ChangeSourceUpstream> | undefined;\n  #lastTx = '';\n  #lastStatus = '';\n  #lastAck = '';\n\n  constructor({trackPgChangeLog, trackBackup}: Opts) {\n    assert(\n      trackPgChangeLog || trackBackup,\n      `At least one of trackPgChangeLog or trackBackup must be true`,\n    );\n    this.#trackPgChangeLog = trackPgChangeLog;\n    this.#trackBackup = trackBackup;\n  }\n\n  /**\n   * Starts tracking a new upstream connection, which resumes the change stream\n   * after `resumeWatermark`.\n   *\n   * Everything at or before `resumeWatermark` was committed on an earlier\n   * connection, so it counts as committed on this one: a status watermark past\n   * it waits for the tracked stores to reach it, as it would for a commit on\n   * this connection. Otherwise the first keepalive of a stream that resumes\n   * ahead of the stores -- a change log ahead of its backup, which is the\n   * normal state -- would move the replication slot past transactions that the\n   * backup has not persisted, and a task restored from the backup would then\n   * resume below the slot, which the upstream moves forward without a word.\n   *\n   * When the PG change log is tracked, the stream resumes from its head, so\n   * `resumeWatermark` has been persisted in it (by this or another\n   * change-streamer). Without this, status messages would not be acked on an\n   * idle upstream until a commit is stored.\n   */\n  reset(upstream: Sink<ChangeSourceUpstream>, resumeWatermark: string) {\n    this.#upstream = upstream;\n    this.#lastTx = resumeWatermark;\n    this.#lastStatus = '';\n    this.#lastAck = '';\n    if (this.#trackPgChangeLog) {\n      this.#pgChangeLogWatermark = max(\n        resumeWatermark,\n        this.#pgChangeLogWatermark,\n      );\n    }\n    // Re-ack what the stores have persisted on the new connection.\n    this.#maybeAck();\n  }\n\n  trackDownstream(downstream: ChangeStreamMessage) {\n    const [tag, msg] = downstream;\n    switch (tag) {\n      case 'status':\n        if (msg.ack) {\n          this.#lastStatus = downstream[2].watermark;\n        }\n        this.#maybeAck();\n        break;\n      case 'commit':\n        // A commit replayed from below the resume watermark does not lower it.\n        this.#lastTx = max(this.#lastTx, downstream[2].watermark);\n        this.#maybeAck();\n        break;\n    }\n  }\n\n  trackPgChangeLog(committedWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#pgChangeLogWatermark = max(\n      committedWatermark,\n      this.#pgChangeLogWatermark,\n    );\n    this.#maybeAck();\n  }\n\n  trackBackup(backedUpWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#backupWatermark = max(backedUpWatermark, this.#backupWatermark);\n    this.#maybeAck();\n  }\n\n  #maybeAck() {\n    const currentWatermark = !this.#trackBackup\n      ? this.#pgChangeLogWatermark\n      : !this.#trackPgChangeLog\n        ? this.#backupWatermark\n        : min(this.#pgChangeLogWatermark, this.#backupWatermark);\n    if (currentWatermark > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {tag: 'commit'},\n        {watermark: currentWatermark},\n      ]);\n      this.#lastAck = currentWatermark;\n    }\n    // If all committed transactions have been acked, ack any outstanding\n    // status LSN's thereafter (i.e. non-publication upstream changes).\n    if (this.#lastAck >= this.#lastTx && this.#lastStatus > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {ack: true},\n        {watermark: this.#lastStatus},\n      ]);\n      this.#lastAck = this.#lastStatus;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+BA,IAAa,gBAAb,MAA2B;CACzB;CACA;CAEA,wBAAwB;CACxB,mBAAmB;CAEnB;CACA,UAAU;CACV,cAAc;CACd,WAAW;CAEX,YAAY,EAAC,kBAAkB,eAAoB;EACjD,OACE,oBAAoB,aACpB,8DACF;EACA,KAAK,oBAAoB;EACzB,KAAK,eAAe;CACtB;;;;;;;;;;;;;;;;;;;CAoBA,MAAM,UAAsC,iBAAyB;EACnE,KAAK,YAAY;EACjB,KAAK,UAAU;EACf,KAAK,cAAc;EACnB,KAAK,WAAW;EAChB,IAAI,KAAK,mBACP,KAAK,wBAAwB,IAC3B,iBACA,KAAK,qBACP;EAGF,KAAK,UAAU;CACjB;CAEA,gBAAgB,YAAiC;EAC/C,MAAM,CAAC,KAAK,OAAO;EACnB,QAAQ,KAAR;GACE,KAAK;IACH,IAAI,IAAI,KACN,KAAK,cAAc,WAAW,EAAE,CAAC;IAEnC,KAAK,UAAU;IACf;GACF,KAAK;IAEH,KAAK,UAAU,IAAI,KAAK,SAAS,WAAW,EAAE,CAAC,SAAS;IACxD,KAAK,UAAU;EAEnB;CACF;CAEA,iBAAiB,oBAA4B;EAE3C,KAAK,wBAAwB,IAC3B,oBACA,KAAK,qBACP;EACA,KAAK,UAAU;CACjB;CAEA,YAAY,mBAA2B;EAErC,KAAK,mBAAmB,IAAI,mBAAmB,KAAK,gBAAgB;EACpE,KAAK,UAAU;CACjB;CAEA,YAAY;EACV,MAAM,mBAAmB,CAAC,KAAK,eAC3B,KAAK,wBACL,CAAC,KAAK,oBACJ,KAAK,mBACL,IAAI,KAAK,uBAAuB,KAAK,gBAAgB;EAC3D,IAAI,mBAAmB,KAAK,UAAU;GACpC,KAAK,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,SAAQ;IACd,EAAC,WAAW,iBAAgB;GAC9B,CAAC;GACD,KAAK,WAAW;EAClB;EAGA,IAAI,KAAK,YAAY,KAAK,WAAW,KAAK,cAAc,KAAK,UAAU;GACrE,KAAK,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,KAAI;IACV,EAAC,WAAW,KAAK,YAAW;GAC9B,CAAC;GACD,KAAK,WAAW,KAAK;EACvB;CACF;AACF"}

package/out/zero-cache/src/services/heapz.js

@@ -4,7 +4,8 @@ import v8 from "v8";
 import auth from "basic-auth";
 //#region ../zero-cache/src/services/heapz.ts
 function handleHeapzRequest(lc, config, req, res) {
-	if (!isAdminPasswordValid(lc, config, auth(req)?.pass)) {
+	const credentials = auth(req);
+	if (!isAdminPasswordValid(lc, config, credentials?.pass)) {
 		res.code(401).header("WWW-Authenticate", "Basic realm=\"Heapz Protected Area\"").send("Unauthorized");
 		return;
 	}

package/out/zero-cache/src/services/heapz.js.map

@@ -1 +1 @@
-{"version":3,"file":"heapz.js","names":[],"sources":["../../../../../zero-cache/src/services/heapz.ts"],"sourcesContent":["import fs from 'fs';\nimport v8 from 'v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\n\nexport function handleHeapzRequest(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Heapz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  const filename = v8.writeHeapSnapshot();\n  const stream = fs.createReadStream(filename);\n  void res\n    .header('Content-Type', 'application/octet-stream')\n    .header('Content-Disposition', `attachment; filename=${filename}`)\n    .send(stream);\n\n  // Clean up temp file after streaming\n  stream.on('end', () => {\n    fs.unlink(filename, err => {\n      if (err) {\n        lc.error?.('Error deleting heap snapshot:', err);\n      }\n    });\n  });\n}\n"],"mappings":";;;;;AAQA,SAAgB,mBACd,IACA,QACA,KACA,KACA;CAEA,IAAI,CAAC,qBAAqB,IAAI,QADV,KAAK,GACa,GAAa,IAAI,GAAG;EACxD,IACG,KAAK,GAAG,EACR,OAAO,oBAAoB,sCAAoC,EAC/D,KAAK,cAAc;EACtB;CACF;CAEA,MAAM,WAAW,GAAG,kBAAkB;CACtC,MAAM,SAAS,GAAG,iBAAiB,QAAQ;CAC3C,IACG,OAAO,gBAAgB,0BAA0B,EACjD,OAAO,uBAAuB,wBAAwB,UAAU,EAChE,KAAK,MAAM;CAGd,OAAO,GAAG,aAAa;EACrB,GAAG,OAAO,WAAU,QAAO;GACzB,IAAI,KACF,GAAG,QAAQ,iCAAiC,GAAG;EAEnD,CAAC;CACH,CAAC;AACH"}
+{"version":3,"file":"heapz.js","names":[],"sources":["../../../../../zero-cache/src/services/heapz.ts"],"sourcesContent":["import fs from 'fs';\nimport v8 from 'v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\n\nexport function handleHeapzRequest(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Heapz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  const filename = v8.writeHeapSnapshot();\n  const stream = fs.createReadStream(filename);\n  void res\n    .header('Content-Type', 'application/octet-stream')\n    .header('Content-Disposition', `attachment; filename=${filename}`)\n    .send(stream);\n\n  // Clean up temp file after streaming\n  stream.on('end', () => {\n    fs.unlink(filename, err => {\n      if (err) {\n        lc.error?.('Error deleting heap snapshot:', err);\n      }\n    });\n  });\n}\n"],"mappings":";;;;;AAQA,SAAgB,mBACd,IACA,QACA,KACA,KACA;CACA,MAAM,cAAc,KAAK,GAAG;CAC5B,IAAI,CAAC,qBAAqB,IAAI,QAAQ,aAAa,IAAI,GAAG;EACxD,IACG,KAAK,GAAG,CAAC,CACT,OAAO,oBAAoB,sCAAoC,CAAC,CAChE,KAAK,cAAc;EACtB;CACF;CAEA,MAAM,WAAW,GAAG,kBAAkB;CACtC,MAAM,SAAS,GAAG,iBAAiB,QAAQ;CAC3C,IACG,OAAO,gBAAgB,0BAA0B,CAAC,CAClD,OAAO,uBAAuB,wBAAwB,UAAU,CAAC,CACjE,KAAK,MAAM;CAGd,OAAO,GAAG,aAAa;EACrB,GAAG,OAAO,WAAU,QAAO;GACzB,IAAI,KACF,GAAG,QAAQ,iCAAiC,GAAG;EAEnD,CAAC;CACH,CAAC;AACH"}

package/out/zero-cache/src/services/http-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.js","names":["#fastify","#port","#heartbeatMonitor","#init","#ready","#failHealthChecks"],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n\n  // Wait for the readinessGate to resolve before responding to health checks.\n  readinessGate?: Promise<unknown> | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  #ready = false;\n  #failHealthChecks = false;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n\n    void readinessGate.then(() => (this.#ready = true));\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => {\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  /**\n   * Responds to subsequent health checks (both \"/\" and \"/keepalive\") with a\n   * 503, e.g. to signal the container runner to stop routing to, and shut\n   * down, a server that is draining itself (see {@link SelfDrain}).\n   */\n  failHealthChecks() {\n    this._lc.info?.(`${this.id}: failing health checks to self drain`);\n    this.#failHealthChecks = true;\n  }\n\n  /**\n   * Stops accepting new connections (e.g. when draining), while existing\n   * connections, including websockets, continue to be served until\n   * {@link stop} is called.\n   */\n  stopAccepting() {\n    if (this.#fastify.server.listening) {\n      this._lc.info?.(`${this.id}: no longer accepting new connections`);\n      this.#fastify.server.close();\n    }\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CACT,oBAAoB;CAEpB,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKA,WAAW,QAAQ;EACxB,KAAKC,QAAQ;EACb,KAAKE,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAKD,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAKE,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAKJ,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAKK,mBACP,IAAI,KAAK,GAAG,EAAE,KAAK,UAAU;QACxB,IAAI,KAAKD,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAKJ,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAKE,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAKG,mBACP,IAAI,KAAK,GAAG,EAAE,KAAK,UAAU;QACxB,IAAI,KAAKD,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAKD,MAAM,KAAKH,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAKA,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAKC;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;;;;;;CAOA,mBAAmB;EACjB,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,sCAAsC;EACjE,KAAKI,oBAAoB;CAC3B;;;;;;CAOA,gBAAgB;EACd,IAAI,KAAKL,SAAS,OAAO,WAAW;GAClC,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,sCAAsC;GACjE,KAAKA,SAAS,OAAO,MAAM;EAC7B;CACF;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAKE,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAKF,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}
+{"version":3,"file":"http-service.js","names":[],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n\n  // Wait for the readinessGate to resolve before responding to health checks.\n  readinessGate?: Promise<unknown> | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  #ready = false;\n  #failHealthChecks = false;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n\n    void readinessGate.then(() => (this.#ready = true));\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => {\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  /**\n   * Responds to subsequent health checks (both \"/\" and \"/keepalive\") with a\n   * 503, e.g. to signal the container runner to stop routing to, and shut\n   * down, a server that is draining itself (see {@link SelfDrain}).\n   */\n  failHealthChecks() {\n    this._lc.info?.(`${this.id}: failing health checks to self drain`);\n    this.#failHealthChecks = true;\n  }\n\n  /**\n   * Stops accepting new connections (e.g. when draining), while existing\n   * connections, including websockets, continue to be served until\n   * {@link stop} is called.\n   */\n  stopAccepting() {\n    if (this.#fastify.server.listening) {\n      this._lc.info?.(`${this.id}: no longer accepting new connections`);\n      this.#fastify.server.close();\n    }\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CACT,oBAAoB;CAEpB,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAK,WAAW,QAAQ;EACxB,KAAK,QAAQ;EACb,KAAK,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAK,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAK,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAK,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAK,mBACP,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU;QACxB,IAAI,KAAK,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAK,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAK,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAK,mBACP,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU;QACxB,IAAI,KAAK,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAK,MAAM,KAAK,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAK,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAK;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;;;;;;CAOA,mBAAmB;EACjB,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,sCAAsC;EACjE,KAAK,oBAAoB;CAC3B;;;;;;CAOA,gBAAgB;EACd,IAAI,KAAK,SAAS,OAAO,WAAW;GAClC,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,sCAAsC;GACjE,KAAK,SAAS,OAAO,MAAM;EAC7B;CACF;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAK,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAK,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}

package/out/zero-cache/src/services/life-cycle.js

@@ -1,8 +1,8 @@
 import { AbortError } from "../../../shared/src/abort-error.js";
 import { singleProcessMode } from "../types/processes.js";
-import { LONG_DURATION_HISTOGRAM_BOUNDARIES_S, getOrCreateHistogram } from "../observability/metrics.js";
 import { ConfigurationError } from "../types/configuration-error.js";
 import { logLastChanceSQLiteCorruptionDiagnostics } from "../db/sqlite-corruption.js";
+import { LONG_DURATION_HISTOGRAM_BOUNDARIES_S, getOrCreateHistogram } from "../observability/metrics.js";
 import { orTimeout } from "../types/timeout.js";
 import { RunningState } from "./running-state.js";
 import { resolver } from "@rocicorp/resolver";
@@ -397,6 +397,6 @@ var HeartbeatMonitor = class {
 	}
 };
 //#endregion
-export { HeartbeatMonitor, ProcessManager, SelfDrain, SupportingProcesses, exitAfter, recordStartupDurationMs, runUntilKilled };
+export { FORCEFUL_SHUTDOWN, GRACEFUL_SHUTDOWN, HeartbeatMonitor, ProcessManager, STOP_TIMEOUT_MS, SelfDrain, SupportingProcesses, exitAfter, recordStartupDurationMs, runUntilKilled };
 
 //# sourceMappingURL=life-cycle.js.map

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#drainStart","#runningState","#onExit","#initializing","#nextID","#parent","#procs","#done","#onClose","#signaled","#proc","#delayMs","#onRequested","#timer","#requested","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {orTimeout} from '../types/timeout.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n *\n * A clean exit (i.e. code 0) of a worker of either type that is not the\n * result of a drain is treated as an intentional shutdown of the server.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a worker exiting cleanly outside of a drain,\n// such as for an auto reset), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    // A clean exit outside of a drain is an intentional shutdown, e.g. the\n    // replication-manager shutting down for protocol-specific instructions\n    // (like auto reset), or a configuration error. In this case, a special\n    // error code is used to signal that the server should be shut down\n    // without draining, but it is otherwise not considered an\n    // unexpected/alertable error. The same applies to a supporting worker\n    // that exits while user-facing workers are still draining.\n    if (\n      code === 0 &&\n      (this.#drainStart === 0 ||\n        (type === 'supporting' && this.#userFacing.size > 0))\n    ) {\n      code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n    }\n\n    if (type === 'supporting') {\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the supporting subprocesses of a worker (e.g. the litestream process\n * of the backup-replicator) as a {@link SingletonService}.\n *\n * Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`\n * to its subprocesses. Instead, {@link runUntilKilled} stops it along with the\n * worker's other services, i.e. after they have drained, so that the\n * subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`\n * and `SIGABRT`), on the other hand, are forwarded immediately once the\n * service is running, and the subprocesses are killed if the worker exits\n * before they are stopped.\n *\n * `run()` settles when a subprocess exits before {@link stop()} is called,\n * which stops the worker: a clean exit resolves (and the parent in turn treats\n * the worker's clean exit outside of a drain as an intentional shutdown),\n * while any other exit rejects.\n */\nexport class SupportingProcesses implements SingletonService {\n  readonly id = 'supporting-processes';\n  readonly #lc: LogContext;\n  readonly #parent: EventEmitter;\n  readonly #procs = new Map<Subprocess, string>();\n  readonly #done = resolver<void>();\n  #signaled: NodeJS.Signals | undefined;\n\n  constructor(lc: LogContext, parent: EventEmitter) {\n    this.#lc = lc.withContext('component', 'supporting-processes');\n    this.#parent = parent;\n    // run() may never be called if the worker fails before running services.\n    this.#done.promise.catch(() => {});\n\n    parent.on('exit', code =>\n      this.#kill(\n        code === 0\n          ? 'SIGTERM'\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT'\n            : 'SIGABRT',\n      ),\n    );\n  }\n\n  add<P extends Subprocess>(proc: P, name: string): P {\n    this.#procs.set(proc, name);\n    proc.on('close', (code: number | null, sig: NodeJS.Signals | null) =>\n      this.#onClose(proc, name, code, sig),\n    );\n    // As with the ProcessManager, 'error' events after a signal (e.g. failing\n    // to send a message to an exiting process) are logged at 'warn'.\n    proc.on('error', err =>\n      this.#lc[this.#signaled ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n    return proc;\n  }\n\n  run(): Promise<void> {\n    // Signal listeners are only installed when running, i.e. alongside those\n    // of runUntilKilled(). Installing them any earlier would consume signals\n    // received during worker initialization, which would otherwise terminate\n    // the worker (or be handled by the initialization logic).\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      this.#parent.on(signal, () => this.#kill(signal));\n    }\n    return this.#done.promise;\n  }\n\n  async stop(): Promise<void> {\n    if (!this.#signaled) {\n      this.#kill('SIGTERM');\n    }\n    if (this.#procs.size === 0) {\n      this.#done.resolve();\n    }\n    await this.#done.promise.catch(() => {});\n  }\n\n  #onClose(\n    proc: Subprocess,\n    name: string,\n    code: number | null,\n    sig: NodeJS.Signals | null,\n  ) {\n    this.#procs.delete(proc);\n    const exit = sig\n      ? `${name} (${proc.pid}) killed with (${sig})`\n      : `${name} (${proc.pid}) exited with code (${code})`;\n\n    if (this.#signaled) {\n      this.#lc.info?.(exit);\n      if (this.#procs.size === 0) {\n        this.#done.resolve();\n      }\n    } else if (code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE) {\n      this.#lc.info?.(exit);\n      this.#done.resolve();\n    } else {\n      this.#done.reject(new Error(exit));\n    }\n  }\n\n  #kill(signal: NodeJS.Signals) {\n    this.#signaled = signal;\n    for (const proc of this.#procs.keys()) {\n      try {\n        proc.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\nexport type SelfDrainOptions = {\n  /**\n   * The time after a self drain is requested at which the server sends itself\n   * a `SIGTERM`, unless one was received sooner (e.g. from the container\n   * runner).\n   */\n  delayMs: number;\n\n  /**\n   * Called when a self drain is requested. This should fail health checks\n   * (both readiness and liveness) so that the container runner stops routing\n   * to the server and initiates its normal shutdown process.\n   */\n  onRequested: () => void;\n};\n\n/**\n * Coordinates a drain that is initiated by the server itself (e.g. by a\n * view-syncer that has been granted permission to restore a new replica),\n * as opposed to one initiated by the container runner with a `SIGTERM`.\n *\n * Health checks start failing immediately, upon which an orchestrator such\n * as Kubernetes stops routing to the server and shuts it down as usual\n * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,\n * and for environments without liveness checks, the server sends itself a\n * `SIGTERM` after the configured delay.\n */\nexport class SelfDrain {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #onRequested: () => void;\n  #requested = false;\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions) {\n    this.#lc = lc.withContext('component', 'self-drain');\n    this.#proc = proc;\n    this.#delayMs = opts.delayMs;\n    this.#onRequested = opts.onRequested;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.once(signal, () => {\n        this.#signaled = true;\n        clearTimeout(this.#timer);\n      });\n    }\n  }\n\n  request() {\n    if (this.#requested || this.#signaled) {\n      return; // Already draining.\n    }\n    this.#requested = true;\n    this.#lc.info?.(\n      `self drain requested. sending SIGTERM in ${this.#delayMs} ms`,\n    );\n    this.#onRequested();\n    this.#timer = setTimeout(() => {\n      if (!this.#signaled) {\n        this.#lc.info?.(`initiating self drain`);\n        this.#proc.emit('SIGTERM', 'SIGTERM');\n      }\n    }, this.#delayMs);\n  }\n}\n\n/**\n * The maximum time that {@link runUntilKilled} waits for its services to stop\n * (i.e. for their `run()` to settle) after stopping them.\n */\nexport const STOP_TIMEOUT_MS = 60_000;\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`, or when the parent process disconnects.\n * Repeated graceful signals are ignored. If a service stops (or throws) on its\n * own, the remaining services are stopped without draining.\n *\n * @returns a Promise that settles once all services have stopped (or\n *          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting\n *          with the outcome of the first service to stop.\n */\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  const stopped = new Set<SingletonService>();\n  const runs = services.map(svc => svc.run().finally(() => stopped.add(svc)));\n\n  const allStopped = resolver<void>();\n  let stopping: Promise<void> | undefined;\n  const stopAll = (reason: string) =>\n    (stopping ??= stopServices(lc, services, runs, stopped, reason).then(\n      allStopped.resolve,\n    ));\n\n  const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n  let draining = false;\n  const onSignal = async (signal: NodeJS.Signals) => {\n    if (GRACEFUL_SIGNALS.includes(signal)) {\n      if (draining || stopping) {\n        // e.g. A SIGINT from both the terminal and a wrapping process.\n        lc.info?.(`already shutting down. ignoring ${signal}`);\n        return;\n      }\n      draining = true;\n      // Drain all services before stopping any of them. Stopping a service\n      // resolves its run() and thus ends runUntilKilled(), so a service\n      // without a drain() must not be stopped while others are draining.\n      const results = await Promise.allSettled(\n        services.map(async svc => {\n          if (svc.drain) {\n            lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n            await svc.drain();\n          }\n        }),\n      );\n      for (const result of results) {\n        if (result.status === 'rejected') {\n          lc.warn?.(`error while draining (${signal})`, result.reason);\n        }\n      }\n    }\n    // A forceful signal stops the services without waiting for (or after)\n    // a drain.\n    await stopAll(signal);\n  };\n  // The listeners remain installed for the lifetime of the process, as a\n  // signal without a listener would terminate the process (e.g. a repeated\n  // SIGINT during a drain).\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.on(signal, () => void onSignal(signal));\n  }\n  // A forked worker whose parent has exited (e.g. was killed by a signal,\n  // in which case it cannot signal its workers) stops without draining, so\n  // that it (and in turn its own subprocesses) is not orphaned.\n  parent.once('disconnect', () => {\n    lc.warn?.(`parent process disconnected. stopping without draining`);\n    void stopAll('disconnect');\n  });\n\n  try {\n    // Run all services and resolve when any of them stops, or when stopping\n    // them (after a signal) has timed out.\n    const svc = await Promise.race([\n      ...runs.map((run, i) => run.then(() => services[i])),\n      allStopped.promise,\n    ]);\n    if (svc) {\n      lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n    }\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  } finally {\n    await stopAll('stopped');\n  }\n}\n\nasync function stopServices(\n  lc: LogContext,\n  services: SingletonService[],\n  runs: Promise<void>[],\n  stopped: Set<SingletonService>,\n  reason: string,\n) {\n  for (const svc of services) {\n    if (!stopped.has(svc)) {\n      lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${reason})`);\n      void svc\n        .stop()\n        .catch(e =>\n          lc.warn?.(`error stopping ${svc.constructor.name} ${svc.id}`, e),\n        );\n    }\n  }\n  if (\n    (await orTimeout(Promise.allSettled(runs), STOP_TIMEOUT_MS)) === 'timed-out'\n  ) {\n    const running = services.filter(svc => !stopped.has(svc));\n    lc.warn?.(\n      `services did not stop within ${STOP_TIMEOUT_MS} ms: ` +\n        running.map(svc => `${svc.constructor.name} ${svc.id}`).join(', '),\n    );\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;AAuCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,EAAE,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAKM,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAKC,MACH,KAAKL,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAKM,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAKL,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAKM,gBAAgB,GACvB,KAAKH,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAKI,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAKV,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAKU,cAAc,KAAK,KAAKV,GAAG;EAChC,KAAUA,IAAI,MAAM,EAAE,cAAc,KAAKG,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAKD,KAAK,SAAS,GAAG;GAExB,KAAKF,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAKQ,MAAM,CAAC;EACd;EACA,KAAKR,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAKS,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAKR,YAAY,MACnB,KAAKM,MAAM,KAAKN,aAAa,MAAM;OAEnC,KAAKM,MAAM,KAAKL,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAKD,YAAY,IAAI,IAAI;EAE3B,KAAKC,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAKS,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAKX,IAAI,CAAC,UAAU,KAAKS,cAAc,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAKI;EAClB,KAAKD,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAKP,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,EAAE,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAKL,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAKI,OAAO,KAAK;GACjE,KAAKQ,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAKA,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAKP,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAKM,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAKV,YAAY,OAAO,MAAM;GAC9B,KAAKC,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EASnC,IACE,SAAS,MACR,KAAKO,gBAAgB,KACnB,SAAS,gBAAgB,KAAKR,YAAY,OAAO,IAEpD,OAAA;EAGF,IAAI,SAAS,cAAc;GACzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAKD,IAAI,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAKQ,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAKC,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAKT,IAAI,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAKC,YAAY,SAAS,GAAG;GAC/B,KAAKD,IAAI,OACP,KAAKS,cACD,oCACE,KAAK,IAAI,IAAI,KAAKA,YACnB,QACD,gCACN;GACA,OAAO,KAAKD,MAAM,KAAKC,cAAc,IAAI,IAAI;EAC/C;EAEA,IAAI,KAAKA,gBAAgB,GAGvB,OAAO,KAAKD,MAAM,IAAI;CAI1B;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAKR,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,sBAAb,MAA6D;CAC3D,KAAc;CACd;CACA;CACA,yBAAkB,IAAI,IAAwB;CAC9C,QAAiB,SAAe;CAChC;CAEA,YAAY,IAAgB,QAAsB;EAChD,KAAKA,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAKc,UAAU;EAEf,KAAKE,MAAM,QAAQ,YAAY,CAAC,CAAC;EAEjC,OAAO,GAAG,SAAQ,SAChB,KAAKT,MACH,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;CACF;CAEA,IAA0B,MAAS,MAAiB;EAClD,KAAKQ,OAAO,IAAI,MAAM,IAAI;EAC1B,KAAK,GAAG,UAAU,MAAqB,QACrC,KAAKE,SAAS,MAAM,MAAM,MAAM,GAAG,CACrC;EAGA,KAAK,GAAG,UAAS,QACf,KAAKjB,IAAI,KAAKkB,YAAY,SAAS,WACjC,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;EACA,OAAO;CACT;CAEA,MAAqB;EAKnB,KAAK,MAAM,UAAU,mBACnB,KAAKJ,QAAQ,GAAG,cAAc,KAAKP,MAAM,MAAM,CAAC;EAElD,OAAO,KAAKS,MAAM;CACpB;CAEA,MAAM,OAAsB;EAC1B,IAAI,CAAC,KAAKE,WACR,KAAKX,MAAM,SAAS;EAEtB,IAAI,KAAKQ,OAAO,SAAS,GACvB,KAAKC,MAAM,QAAQ;EAErB,MAAM,KAAKA,MAAM,QAAQ,YAAY,CAAC,CAAC;CACzC;CAEA,SACE,MACA,MACA,MACA,KACA;EACA,KAAKD,OAAO,OAAO,IAAI;EACvB,MAAM,OAAO,MACT,GAAG,KAAK,IAAI,KAAK,IAAI,iBAAiB,IAAI,KAC1C,GAAG,KAAK,IAAI,KAAK,IAAI,sBAAsB,KAAK;EAEpD,IAAI,KAAKG,WAAW;GAClB,KAAKlB,IAAI,OAAO,IAAI;GACpB,IAAI,KAAKe,OAAO,SAAS,GACvB,KAAKC,MAAM,QAAQ;EAEvB,OAAO,IAAI,SAAS,KAAK,SAAA,IAA0C;GACjE,KAAKhB,IAAI,OAAO,IAAI;GACpB,KAAKgB,MAAM,QAAQ;EACrB,OACE,KAAKA,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC;CAErC;CAEA,MAAM,QAAwB;EAC5B,KAAKE,YAAY;EACjB,KAAK,MAAM,QAAQ,KAAKH,OAAO,KAAK,GAClC,IAAI;GACF,KAAK,KAAK,MAAM;EAClB,SAAS,GAAG;GACV,KAAKf,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;AA6BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CACA,aAAa;CACb,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,MAAwB;EACtE,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKmB,QAAQ;EACb,KAAKC,WAAW,KAAK;EACrB,KAAKC,eAAe,KAAK;EAEzB,KAAK,MAAM,UAAU,mBACnB,KAAK,KAAK,cAAc;GACtB,KAAKH,YAAY;GACjB,aAAa,KAAKI,MAAM;EAC1B,CAAC;CAEL;CAEA,UAAU;EACR,IAAI,KAAKC,cAAc,KAAKL,WAC1B;EAEF,KAAKK,aAAa;EAClB,KAAKvB,IAAI,OACP,4CAA4C,KAAKoB,SAAS,IAC5D;EACA,KAAKC,aAAa;EAClB,KAAKC,SAAS,iBAAiB;GAC7B,IAAI,CAAC,KAAKJ,WAAW;IACnB,KAAKlB,IAAI,OAAO,uBAAuB;IACvC,KAAKmB,MAAM,KAAK,WAAW,SAAS;GACtC;EACF,GAAG,KAAKC,QAAQ;CAClB;AACF;;;;;AAMA,IAAa,kBAAkB;;;;;;;;;;;;AAa/B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,MAAM,0BAAU,IAAI,IAAsB;CAC1C,MAAM,OAAO,SAAS,KAAI,QAAO,IAAI,IAAI,EAAE,cAAc,QAAQ,IAAI,GAAG,CAAC,CAAC;CAE1E,MAAM,aAAa,SAAe;CAClC,IAAI;CACJ,MAAM,WAAW,WACd,aAAa,aAAa,IAAI,UAAU,MAAM,SAAS,MAAM,EAAE,KAC9D,WAAW,OACb;CAEF,MAAM,mBAAmB;CACzB,IAAI,WAAW;CACf,MAAM,WAAW,OAAO,WAA2B;EACjD,IAAI,iBAAiB,SAAS,MAAM,GAAG;GACrC,IAAI,YAAY,UAAU;IAExB,GAAG,OAAO,mCAAmC,QAAQ;IACrD;GACF;GACA,WAAW;GAIX,MAAM,UAAU,MAAM,QAAQ,WAC5B,SAAS,IAAI,OAAM,QAAO;IACxB,IAAI,IAAI,OAAO;KACb,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;KAClE,MAAM,IAAI,MAAM;IAClB;GACF,CAAC,CACH;GACA,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,WAAW,YACpB,GAAG,OAAO,yBAAyB,OAAO,IAAI,OAAO,MAAM;EAGjE;EAGA,MAAM,QAAQ,MAAM;CACtB;CAIA,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,GAAG,cAAc,KAAK,SAAS,MAAM,CAAC;CAK/C,OAAO,KAAK,oBAAoB;EAC9B,GAAG,OAAO,wDAAwD;EAClE,QAAa,YAAY;CAC3B,CAAC;CAED,IAAI;EAGF,MAAM,MAAM,MAAM,QAAQ,KAAK,CAC7B,GAAG,KAAK,KAAK,KAAK,MAAM,IAAI,WAAW,SAAS,EAAE,CAAC,GACnD,WAAW,OACb,CAAC;EACD,IAAI,KACF,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CAE3D,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR,UAAU;EACR,MAAM,QAAQ,SAAS;CACzB;AACF;AAEA,eAAe,aACb,IACA,UACA,MACA,SACA,QACA;CACA,KAAK,MAAM,OAAO,UAChB,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG;EACrB,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;EAClE,IACG,KAAK,EACL,OAAM,MACL,GAAG,OAAO,kBAAkB,IAAI,YAAY,KAAK,GAAG,IAAI,MAAM,CAAC,CACjE;CACJ;CAEF,IACG,MAAM,UAAU,QAAQ,WAAW,IAAI,GAAA,GAAkB,MAAO,aACjE;EACA,MAAM,UAAU,SAAS,QAAO,QAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;EACxD,GAAG,OACD,gCAAgC,gBAAgB,SAC9C,QAAQ,KAAI,QAAO,GAAG,IAAI,YAAY,KAAK,GAAG,IAAI,IAAI,EAAE,KAAK,IAAI,CACrE;CACF;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,EAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAKpB,MAAM;EACX,KAAKwB,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAKC,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAKC,wBAAwB,KAAA,GAAW;GAC1C,KAAK1B,IAAI,OACP,iCACE,KAAKwB,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAKE,sBAAsB,YACzB,KAAKC,oBACL,KAAKH,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAKI,uBAAuB,mBAAmB;GAC7C,KAAKA,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAKH;GACjD,IAAI,0BAA0B,KAAKD,eAAe;IAChD,KAAKxB,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAK0B,mBAAmB;EACrC,IAAI,KAAKE,sBACP,eAAe,KAAKA,oBAAoB;CAE5C;AACF"}
+{"version":3,"file":"life-cycle.js","names":[],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {orTimeout} from '../types/timeout.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n *\n * A clean exit (i.e. code 0) of a worker of either type that is not the\n * result of a drain is treated as an intentional shutdown of the server.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a worker exiting cleanly outside of a drain,\n// such as for an auto reset), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    // A clean exit outside of a drain is an intentional shutdown, e.g. the\n    // replication-manager shutting down for protocol-specific instructions\n    // (like auto reset), or a configuration error. In this case, a special\n    // error code is used to signal that the server should be shut down\n    // without draining, but it is otherwise not considered an\n    // unexpected/alertable error. The same applies to a supporting worker\n    // that exits while user-facing workers are still draining.\n    if (\n      code === 0 &&\n      (this.#drainStart === 0 ||\n        (type === 'supporting' && this.#userFacing.size > 0))\n    ) {\n      code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n    }\n\n    if (type === 'supporting') {\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the supporting subprocesses of a worker (e.g. the litestream process\n * of the backup-replicator) as a {@link SingletonService}.\n *\n * Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`\n * to its subprocesses. Instead, {@link runUntilKilled} stops it along with the\n * worker's other services, i.e. after they have drained, so that the\n * subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`\n * and `SIGABRT`), on the other hand, are forwarded immediately once the\n * service is running, and the subprocesses are killed if the worker exits\n * before they are stopped.\n *\n * `run()` settles when a subprocess exits before {@link stop()} is called,\n * which stops the worker: a clean exit resolves (and the parent in turn treats\n * the worker's clean exit outside of a drain as an intentional shutdown),\n * while any other exit rejects.\n */\nexport class SupportingProcesses implements SingletonService {\n  readonly id = 'supporting-processes';\n  readonly #lc: LogContext;\n  readonly #parent: EventEmitter;\n  readonly #procs = new Map<Subprocess, string>();\n  readonly #done = resolver<void>();\n  #signaled: NodeJS.Signals | undefined;\n\n  constructor(lc: LogContext, parent: EventEmitter) {\n    this.#lc = lc.withContext('component', 'supporting-processes');\n    this.#parent = parent;\n    // run() may never be called if the worker fails before running services.\n    this.#done.promise.catch(() => {});\n\n    parent.on('exit', code =>\n      this.#kill(\n        code === 0\n          ? 'SIGTERM'\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT'\n            : 'SIGABRT',\n      ),\n    );\n  }\n\n  add<P extends Subprocess>(proc: P, name: string): P {\n    this.#procs.set(proc, name);\n    proc.on('close', (code: number | null, sig: NodeJS.Signals | null) =>\n      this.#onClose(proc, name, code, sig),\n    );\n    // As with the ProcessManager, 'error' events after a signal (e.g. failing\n    // to send a message to an exiting process) are logged at 'warn'.\n    proc.on('error', err =>\n      this.#lc[this.#signaled ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n    return proc;\n  }\n\n  run(): Promise<void> {\n    // Signal listeners are only installed when running, i.e. alongside those\n    // of runUntilKilled(). Installing them any earlier would consume signals\n    // received during worker initialization, which would otherwise terminate\n    // the worker (or be handled by the initialization logic).\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      this.#parent.on(signal, () => this.#kill(signal));\n    }\n    return this.#done.promise;\n  }\n\n  async stop(): Promise<void> {\n    if (!this.#signaled) {\n      this.#kill('SIGTERM');\n    }\n    if (this.#procs.size === 0) {\n      this.#done.resolve();\n    }\n    await this.#done.promise.catch(() => {});\n  }\n\n  #onClose(\n    proc: Subprocess,\n    name: string,\n    code: number | null,\n    sig: NodeJS.Signals | null,\n  ) {\n    this.#procs.delete(proc);\n    const exit = sig\n      ? `${name} (${proc.pid}) killed with (${sig})`\n      : `${name} (${proc.pid}) exited with code (${code})`;\n\n    if (this.#signaled) {\n      this.#lc.info?.(exit);\n      if (this.#procs.size === 0) {\n        this.#done.resolve();\n      }\n    } else if (code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE) {\n      this.#lc.info?.(exit);\n      this.#done.resolve();\n    } else {\n      this.#done.reject(new Error(exit));\n    }\n  }\n\n  #kill(signal: NodeJS.Signals) {\n    this.#signaled = signal;\n    for (const proc of this.#procs.keys()) {\n      try {\n        proc.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\nexport type SelfDrainOptions = {\n  /**\n   * The time after a self drain is requested at which the server sends itself\n   * a `SIGTERM`, unless one was received sooner (e.g. from the container\n   * runner).\n   */\n  delayMs: number;\n\n  /**\n   * Called when a self drain is requested. This should fail health checks\n   * (both readiness and liveness) so that the container runner stops routing\n   * to the server and initiates its normal shutdown process.\n   */\n  onRequested: () => void;\n};\n\n/**\n * Coordinates a drain that is initiated by the server itself (e.g. by a\n * view-syncer that has been granted permission to restore a new replica),\n * as opposed to one initiated by the container runner with a `SIGTERM`.\n *\n * Health checks start failing immediately, upon which an orchestrator such\n * as Kubernetes stops routing to the server and shuts it down as usual\n * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,\n * and for environments without liveness checks, the server sends itself a\n * `SIGTERM` after the configured delay.\n */\nexport class SelfDrain {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #onRequested: () => void;\n  #requested = false;\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions) {\n    this.#lc = lc.withContext('component', 'self-drain');\n    this.#proc = proc;\n    this.#delayMs = opts.delayMs;\n    this.#onRequested = opts.onRequested;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.once(signal, () => {\n        this.#signaled = true;\n        clearTimeout(this.#timer);\n      });\n    }\n  }\n\n  request() {\n    if (this.#requested || this.#signaled) {\n      return; // Already draining.\n    }\n    this.#requested = true;\n    this.#lc.info?.(\n      `self drain requested. sending SIGTERM in ${this.#delayMs} ms`,\n    );\n    this.#onRequested();\n    this.#timer = setTimeout(() => {\n      if (!this.#signaled) {\n        this.#lc.info?.(`initiating self drain`);\n        this.#proc.emit('SIGTERM', 'SIGTERM');\n      }\n    }, this.#delayMs);\n  }\n}\n\n/**\n * The maximum time that {@link runUntilKilled} waits for its services to stop\n * (i.e. for their `run()` to settle) after stopping them.\n */\nexport const STOP_TIMEOUT_MS = 60_000;\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`, or when the parent process disconnects.\n * Repeated graceful signals are ignored. If a service stops (or throws) on its\n * own, the remaining services are stopped without draining.\n *\n * @returns a Promise that settles once all services have stopped (or\n *          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting\n *          with the outcome of the first service to stop.\n */\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  const stopped = new Set<SingletonService>();\n  const runs = services.map(svc => svc.run().finally(() => stopped.add(svc)));\n\n  const allStopped = resolver<void>();\n  let stopping: Promise<void> | undefined;\n  const stopAll = (reason: string) =>\n    (stopping ??= stopServices(lc, services, runs, stopped, reason).then(\n      allStopped.resolve,\n    ));\n\n  const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n  let draining = false;\n  const onSignal = async (signal: NodeJS.Signals) => {\n    if (GRACEFUL_SIGNALS.includes(signal)) {\n      if (draining || stopping) {\n        // e.g. A SIGINT from both the terminal and a wrapping process.\n        lc.info?.(`already shutting down. ignoring ${signal}`);\n        return;\n      }\n      draining = true;\n      // Drain all services before stopping any of them. Stopping a service\n      // resolves its run() and thus ends runUntilKilled(), so a service\n      // without a drain() must not be stopped while others are draining.\n      const results = await Promise.allSettled(\n        services.map(async svc => {\n          if (svc.drain) {\n            lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n            await svc.drain();\n          }\n        }),\n      );\n      for (const result of results) {\n        if (result.status === 'rejected') {\n          lc.warn?.(`error while draining (${signal})`, result.reason);\n        }\n      }\n    }\n    // A forceful signal stops the services without waiting for (or after)\n    // a drain.\n    await stopAll(signal);\n  };\n  // The listeners remain installed for the lifetime of the process, as a\n  // signal without a listener would terminate the process (e.g. a repeated\n  // SIGINT during a drain).\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.on(signal, () => void onSignal(signal));\n  }\n  // A forked worker whose parent has exited (e.g. was killed by a signal,\n  // in which case it cannot signal its workers) stops without draining, so\n  // that it (and in turn its own subprocesses) is not orphaned.\n  parent.once('disconnect', () => {\n    lc.warn?.(`parent process disconnected. stopping without draining`);\n    void stopAll('disconnect');\n  });\n\n  try {\n    // Run all services and resolve when any of them stops, or when stopping\n    // them (after a signal) has timed out.\n    const svc = await Promise.race([\n      ...runs.map((run, i) => run.then(() => services[i])),\n      allStopped.promise,\n    ]);\n    if (svc) {\n      lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n    }\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  } finally {\n    await stopAll('stopped');\n  }\n}\n\nasync function stopServices(\n  lc: LogContext,\n  services: SingletonService[],\n  runs: Promise<void>[],\n  stopped: Set<SingletonService>,\n  reason: string,\n) {\n  for (const svc of services) {\n    if (!stopped.has(svc)) {\n      lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${reason})`);\n      void svc\n        .stop()\n        .catch(e =>\n          lc.warn?.(`error stopping ${svc.constructor.name} ${svc.id}`, e),\n        );\n    }\n  }\n  if (\n    (await orTimeout(Promise.allSettled(runs), STOP_TIMEOUT_MS)) === 'timed-out'\n  ) {\n    const running = services.filter(svc => !stopped.has(svc));\n    lc.warn?.(\n      `services did not stop within ${STOP_TIMEOUT_MS} ms: ` +\n        running.map(svc => `${svc.constructor.name} ${svc.id}`).join(', '),\n    );\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;AAuCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,CAAC,CAAC,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAK,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAK,MACH,KAAK,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAK,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAK,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAK,gBAAgB,GACvB,KAAK,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAK,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAK,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAK,cAAc,KAAK,KAAK,GAAG;EAChC,KAAU,IAAI,MAAM,CAAC,CAAC,cAAc,KAAK,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAK,KAAK,SAAS,GAAG;GAExB,KAAK,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAK,MAAM,CAAC;EACd;EACA,KAAK,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAK,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAK,YAAY,MACnB,KAAK,MAAM,KAAK,aAAa,MAAM;OAEnC,KAAK,MAAM,KAAK,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAK,YAAY,IAAI,IAAI;EAE3B,KAAK,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAK,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAK,IAAI,CAAC,UAAU,KAAK,cAAc,IAAI,SAAS,QAAQ,GAC1D,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAK;EAClB,KAAK,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAK,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,CAAC,CAAC,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAK,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAK,OAAO,KAAK;GACjE,KAAK,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAK,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAK,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAK,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAK,YAAY,OAAO,MAAM;GAC9B,KAAK,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EASnC,IACE,SAAS,MACR,KAAK,gBAAgB,KACnB,SAAS,gBAAgB,KAAK,YAAY,OAAO,IAEpD,OAAA;EAGF,IAAI,SAAS,cAAc;GACzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAK,IAAI,IAAI,GAAG,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAK,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAK,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAK,IAAI,IAAI,GACX,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAK,YAAY,SAAS,GAAG;GAC/B,KAAK,IAAI,OACP,KAAK,cACD,oCACE,KAAK,IAAI,IAAI,KAAK,YACnB,QACD,gCACN;GACA,OAAO,KAAK,MAAM,KAAK,cAAc,IAAI,IAAI;EAC/C;EAEA,IAAI,KAAK,gBAAgB,GAGvB,OAAO,KAAK,MAAM,IAAI;CAI1B;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,sBAAb,MAA6D;CAC3D,KAAc;CACd;CACA;CACA,yBAAkB,IAAI,IAAwB;CAC9C,QAAiB,SAAe;CAChC;CAEA,YAAY,IAAgB,QAAsB;EAChD,KAAK,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAK,UAAU;EAEf,KAAK,MAAM,QAAQ,YAAY,CAAC,CAAC;EAEjC,OAAO,GAAG,SAAQ,SAChB,KAAK,MACH,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;CACF;CAEA,IAA0B,MAAS,MAAiB;EAClD,KAAK,OAAO,IAAI,MAAM,IAAI;EAC1B,KAAK,GAAG,UAAU,MAAqB,QACrC,KAAK,SAAS,MAAM,MAAM,MAAM,GAAG,CACrC;EAGA,KAAK,GAAG,UAAS,QACf,KAAK,IAAI,KAAK,YAAY,SAAS,QAAQ,GACzC,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;EACA,OAAO;CACT;CAEA,MAAqB;EAKnB,KAAK,MAAM,UAAU,mBACnB,KAAK,QAAQ,GAAG,cAAc,KAAK,MAAM,MAAM,CAAC;EAElD,OAAO,KAAK,MAAM;CACpB;CAEA,MAAM,OAAsB;EAC1B,IAAI,CAAC,KAAK,WACR,KAAK,MAAM,SAAS;EAEtB,IAAI,KAAK,OAAO,SAAS,GACvB,KAAK,MAAM,QAAQ;EAErB,MAAM,KAAK,MAAM,QAAQ,YAAY,CAAC,CAAC;CACzC;CAEA,SACE,MACA,MACA,MACA,KACA;EACA,KAAK,OAAO,OAAO,IAAI;EACvB,MAAM,OAAO,MACT,GAAG,KAAK,IAAI,KAAK,IAAI,iBAAiB,IAAI,KAC1C,GAAG,KAAK,IAAI,KAAK,IAAI,sBAAsB,KAAK;EAEpD,IAAI,KAAK,WAAW;GAClB,KAAK,IAAI,OAAO,IAAI;GACpB,IAAI,KAAK,OAAO,SAAS,GACvB,KAAK,MAAM,QAAQ;EAEvB,OAAO,IAAI,SAAS,KAAK,SAAA,IAA0C;GACjE,KAAK,IAAI,OAAO,IAAI;GACpB,KAAK,MAAM,QAAQ;EACrB,OACE,KAAK,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC;CAErC;CAEA,MAAM,QAAwB;EAC5B,KAAK,YAAY;EACjB,KAAK,MAAM,QAAQ,KAAK,OAAO,KAAK,GAClC,IAAI;GACF,KAAK,KAAK,MAAM;EAClB,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;AA6BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CACA,aAAa;CACb,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,MAAwB;EACtE,KAAK,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAK,QAAQ;EACb,KAAK,WAAW,KAAK;EACrB,KAAK,eAAe,KAAK;EAEzB,KAAK,MAAM,UAAU,mBACnB,KAAK,KAAK,cAAc;GACtB,KAAK,YAAY;GACjB,aAAa,KAAK,MAAM;EAC1B,CAAC;CAEL;CAEA,UAAU;EACR,IAAI,KAAK,cAAc,KAAK,WAC1B;EAEF,KAAK,aAAa;EAClB,KAAK,IAAI,OACP,4CAA4C,KAAK,SAAS,IAC5D;EACA,KAAK,aAAa;EAClB,KAAK,SAAS,iBAAiB;GAC7B,IAAI,CAAC,KAAK,WAAW;IACnB,KAAK,IAAI,OAAO,uBAAuB;IACvC,KAAK,MAAM,KAAK,WAAW,SAAS;GACtC;EACF,GAAG,KAAK,QAAQ;CAClB;AACF;;;;;AAMA,IAAa,kBAAkB;;;;;;;;;;;;AAa/B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,MAAM,0BAAU,IAAI,IAAsB;CAC1C,MAAM,OAAO,SAAS,KAAI,QAAO,IAAI,IAAI,CAAC,CAAC,cAAc,QAAQ,IAAI,GAAG,CAAC,CAAC;CAE1E,MAAM,aAAa,SAAe;CAClC,IAAI;CACJ,MAAM,WAAW,WACd,aAAa,aAAa,IAAI,UAAU,MAAM,SAAS,MAAM,CAAC,CAAC,KAC9D,WAAW,OACb;CAEF,MAAM,mBAAmB;CACzB,IAAI,WAAW;CACf,MAAM,WAAW,OAAO,WAA2B;EACjD,IAAI,iBAAiB,SAAS,MAAM,GAAG;GACrC,IAAI,YAAY,UAAU;IAExB,GAAG,OAAO,mCAAmC,QAAQ;IACrD;GACF;GACA,WAAW;GAIX,MAAM,UAAU,MAAM,QAAQ,WAC5B,SAAS,IAAI,OAAM,QAAO;IACxB,IAAI,IAAI,OAAO;KACb,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;KAClE,MAAM,IAAI,MAAM;IAClB;GACF,CAAC,CACH;GACA,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,WAAW,YACpB,GAAG,OAAO,yBAAyB,OAAO,IAAI,OAAO,MAAM;EAGjE;EAGA,MAAM,QAAQ,MAAM;CACtB;CAIA,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,GAAG,cAAc,KAAK,SAAS,MAAM,CAAC;CAK/C,OAAO,KAAK,oBAAoB;EAC9B,GAAG,OAAO,wDAAwD;EAClE,QAAa,YAAY;CAC3B,CAAC;CAED,IAAI;EAGF,MAAM,MAAM,MAAM,QAAQ,KAAK,CAC7B,GAAG,KAAK,KAAK,KAAK,MAAM,IAAI,WAAW,SAAS,EAAE,CAAC,GACnD,WAAW,OACb,CAAC;EACD,IAAI,KACF,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CAE3D,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR,UAAU;EACR,MAAM,QAAQ,SAAS;CACzB;AACF;AAEA,eAAe,aACb,IACA,UACA,MACA,SACA,QACA;CACA,KAAK,MAAM,OAAO,UAChB,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG;EACrB,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;EAClE,IACG,KAAK,CAAC,CACN,OAAM,MACL,GAAG,OAAO,kBAAkB,IAAI,YAAY,KAAK,GAAG,IAAI,MAAM,CAAC,CACjE;CACJ;CAEF,IACG,MAAM,UAAU,QAAQ,WAAW,IAAI,GAAA,GAAkB,MAAO,aACjE;EACA,MAAM,UAAU,SAAS,QAAO,QAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;EACxD,GAAG,OACD,gCAAgC,gBAAgB,SAC9C,QAAQ,KAAI,QAAO,GAAG,IAAI,YAAY,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,KAAK,IAAI,CACrE;CACF;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,CAAA,CAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAK,MAAM;EACX,KAAK,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAK,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAK,wBAAwB,KAAA,GAAW;GAC1C,KAAK,IAAI,OACP,iCACE,KAAK,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAK,sBAAsB,YACzB,KAAK,oBACL,KAAK,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAK,uBAAuB,mBAAmB;GAC7C,KAAK,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAK;GACjD,IAAI,0BAA0B,KAAK,eAAe;IAChD,KAAK,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAK,mBAAmB;EACrC,IAAI,KAAK,sBACP,eAAe,KAAK,oBAAoB;CAE5C;AACF"}

package/out/zero-cache/src/services/limiter/sliding-window-limiter.js

@@ -58,8 +58,10 @@ var SlidingWindowLimiter = class {
 		if (now - this.#windowSizeMs > this.#nextWindow.start + this.#windowSizeMs) [this.#priorWindow, this.#nextWindow] = this.#newWindows();
 		if (now - this.#windowSizeMs >= this.#nextWindow.start) this.#rotateWindows();
 		const canDo = this.totalCallsForTime(now) < this.#maxMutations;
-		if (canDo) if (now < this.#nextWindow.start) this.#priorWindow.count++;
-		else this.#nextWindow.count++;
+		if (canDo) {
+			if (now < this.#nextWindow.start) this.#priorWindow.count++;
+			else this.#nextWindow.count++;
+		}
 		return canDo;
 	}
 	totalCallsForTime(now) {

package/out/zero-cache/src/services/limiter/sliding-window-limiter.js.map

@@ -1 +1 @@
-{"version":3,"file":"sliding-window-limiter.js","names":["#windowSizeMs","#maxMutations","#priorWindow","#nextWindow","#newWindows","#rotateWindows"],"sources":["../../../../../../zero-cache/src/services/limiter/sliding-window-limiter.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\n\ntype Window = {\n  start: number;\n  count: number;\n};\n\n/**\n * The limiter is a sliding window rate limiter that allows a maximum\n * number of mutations per window.\n *\n * The window is divided into two parts: the prev window and the next window.\n * The prev window is the part of the window that has already passed, and the\n * next window is the part of the window that is still in the future.\n *\n * The limiter keeps track of the number of mutations in each window. When the\n * current time moves completely past the next window, the limiter creates new\n * windows (this only happens if the limiter is not called for a long time).\n * When the current time moves completely into the next window, the\n * limiter rotates the windows.\n *\n * Case 1: need new windows\n * |----|----|\n *             |----| (sliding window)\n *\n * Case 2: rotate windows\n * |----|----|\n *      |----| (sliding window)\n *\n * or\n *\n * |----|----|\n *         |----| (sliding window)\n *\n * The leading edge of the sliding window is current time.\n *\n * The limiter increments the count for the prev window if the current time is\n * in the prior window. Otherwise, it increments the count for the next\n * window.\n *\n * The limiter computes the total mutations by: taking the number of mutation in the next window and\n * adding the number of mutation from the prev window, weighted by\n * the fraction of the window that has passed. The total mutations must be less\n * than the maximum number of mutations allowed per window.\n *\n * |----|----|\n *    |----| (sliding window)\n * ^-- the sliding window covers 25% of the prior window. Only 25% of that window's count counts.\n * The entirety of the next window's count counts.\n */\nexport class SlidingWindowLimiter {\n  readonly #windowSizeMs: number;\n  readonly #maxMutations: number;\n\n  #priorWindow: Window;\n  #nextWindow: Window;\n\n  constructor(windowSizeMs: number, maxMutations: number) {\n    this.#windowSizeMs = windowSizeMs;\n    this.#maxMutations = maxMutations;\n\n    [this.#priorWindow, this.#nextWindow] = this.#newWindows();\n  }\n\n  canDo(): boolean {\n    const now = Date.now();\n\n    // If the current sliding window is completely past the next window, we need new windows.\n    if (\n      now - this.#windowSizeMs >\n      this.#nextWindow.start + this.#windowSizeMs\n    ) {\n      [this.#priorWindow, this.#nextWindow] = this.#newWindows();\n    }\n\n    // Has the current sliding window moved completely into the next window?\n    // Then rotate the windows.\n    if (now - this.#windowSizeMs >= this.#nextWindow.start) {\n      this.#rotateWindows();\n    }\n\n    // Now compute the total mutations in the current window.\n    // Weighted by the fraction of the window that has passed.\n    const totalCalls = this.totalCallsForTime(now);\n    const canDo = totalCalls < this.#maxMutations;\n\n    // We don't increment counts if we throttled the user. This is so the user can\n    // eventually do the action they are trying to do. If we incremented on every\n    // attempt they can end up in a case where their excessive retries lock them out\n    // continuously.\n    if (canDo) {\n      // If \"now\" in the prior or next window? Increment the count for the correct window.\n      if (now < this.#nextWindow.start) {\n        this.#priorWindow.count++;\n      } else {\n        this.#nextWindow.count++;\n      }\n    }\n\n    return canDo;\n  }\n\n  totalCallsForTime(now: number): number {\n    let fraction: number;\n    if (now < this.#priorWindow.start + this.#windowSizeMs) {\n      fraction = 1;\n    } else {\n      fraction =\n        (this.#priorWindow.start +\n          (this.#windowSizeMs - 1) -\n          (now - this.#windowSizeMs)) /\n        this.#windowSizeMs;\n    }\n    if (fraction < 0) {\n      fraction = 0;\n    }\n    assert(\n      fraction <= 1,\n      'The past cannot contribute more than the a full window.',\n    );\n    const totalCalls =\n      this.#priorWindow.count * fraction + this.#nextWindow.count;\n    return totalCalls;\n  }\n\n  get priorWindow(): Window {\n    return this.#priorWindow;\n  }\n\n  get nextWindow(): Window {\n    return this.#nextWindow;\n  }\n\n  /**\n   * If there were no mutations in over `windowSizeMs` we need new windows.\n   */\n  #newWindows() {\n    const now = Date.now();\n    const start = now - (now % this.#windowSizeMs);\n    return [\n      {\n        // clamp to a `windowSizeMs` boundary to start the prior window.\n        start,\n        count: 0,\n      },\n      {\n        start: start + this.#windowSizeMs,\n        count: 0,\n      },\n    ];\n  }\n\n  #rotateWindows() {\n    this.#priorWindow = this.#nextWindow;\n    this.#nextWindow = {\n      start: this.#priorWindow.start + this.#windowSizeMs,\n      count: 0,\n    };\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkDA,IAAa,uBAAb,MAAkC;CAChC;CACA;CAEA;CACA;CAEA,YAAY,cAAsB,cAAsB;EACtD,KAAKA,gBAAgB;EACrB,KAAKC,gBAAgB;EAErB,CAAC,KAAKC,cAAc,KAAKC,eAAe,KAAKC,YAAY;CAC3D;CAEA,QAAiB;EACf,MAAM,MAAM,KAAK,IAAI;EAGrB,IACE,MAAM,KAAKJ,gBACX,KAAKG,YAAY,QAAQ,KAAKH,eAE9B,CAAC,KAAKE,cAAc,KAAKC,eAAe,KAAKC,YAAY;EAK3D,IAAI,MAAM,KAAKJ,iBAAiB,KAAKG,YAAY,OAC/C,KAAKE,eAAe;EAMtB,MAAM,QADa,KAAK,kBAAkB,GAC5B,IAAa,KAAKJ;EAMhC,IAAI,OAEF,IAAI,MAAM,KAAKE,YAAY,OACzB,KAAKD,aAAa;OAElB,KAAKC,YAAY;EAIrB,OAAO;CACT;CAEA,kBAAkB,KAAqB;EACrC,IAAI;EACJ,IAAI,MAAM,KAAKD,aAAa,QAAQ,KAAKF,eACvC,WAAW;OAEX,YACG,KAAKE,aAAa,SAChB,KAAKF,gBAAgB,MACrB,MAAM,KAAKA,kBACd,KAAKA;EAET,IAAI,WAAW,GACb,WAAW;EAEb,OACE,YAAY,GACZ,yDACF;EAGA,OADE,KAAKE,aAAa,QAAQ,WAAW,KAAKC,YAAY;CAE1D;CAEA,IAAI,cAAsB;EACxB,OAAO,KAAKD;CACd;CAEA,IAAI,aAAqB;EACvB,OAAO,KAAKC;CACd;;;;CAKA,cAAc;EACZ,MAAM,MAAM,KAAK,IAAI;EACrB,MAAM,QAAQ,MAAO,MAAM,KAAKH;EAChC,OAAO,CACL;GAEE;GACA,OAAO;EACT,GACA;GACE,OAAO,QAAQ,KAAKA;GACpB,OAAO;EACT,CACF;CACF;CAEA,iBAAiB;EACf,KAAKE,eAAe,KAAKC;EACzB,KAAKA,cAAc;GACjB,OAAO,KAAKD,aAAa,QAAQ,KAAKF;GACtC,OAAO;EACT;CACF;AACF"}
+{"version":3,"file":"sliding-window-limiter.js","names":[],"sources":["../../../../../../zero-cache/src/services/limiter/sliding-window-limiter.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\n\ntype Window = {\n  start: number;\n  count: number;\n};\n\n/**\n * The limiter is a sliding window rate limiter that allows a maximum\n * number of mutations per window.\n *\n * The window is divided into two parts: the prev window and the next window.\n * The prev window is the part of the window that has already passed, and the\n * next window is the part of the window that is still in the future.\n *\n * The limiter keeps track of the number of mutations in each window. When the\n * current time moves completely past the next window, the limiter creates new\n * windows (this only happens if the limiter is not called for a long time).\n * When the current time moves completely into the next window, the\n * limiter rotates the windows.\n *\n * Case 1: need new windows\n * |----|----|\n *             |----| (sliding window)\n *\n * Case 2: rotate windows\n * |----|----|\n *      |----| (sliding window)\n *\n * or\n *\n * |----|----|\n *         |----| (sliding window)\n *\n * The leading edge of the sliding window is current time.\n *\n * The limiter increments the count for the prev window if the current time is\n * in the prior window. Otherwise, it increments the count for the next\n * window.\n *\n * The limiter computes the total mutations by: taking the number of mutation in the next window and\n * adding the number of mutation from the prev window, weighted by\n * the fraction of the window that has passed. The total mutations must be less\n * than the maximum number of mutations allowed per window.\n *\n * |----|----|\n *    |----| (sliding window)\n * ^-- the sliding window covers 25% of the prior window. Only 25% of that window's count counts.\n * The entirety of the next window's count counts.\n */\nexport class SlidingWindowLimiter {\n  readonly #windowSizeMs: number;\n  readonly #maxMutations: number;\n\n  #priorWindow: Window;\n  #nextWindow: Window;\n\n  constructor(windowSizeMs: number, maxMutations: number) {\n    this.#windowSizeMs = windowSizeMs;\n    this.#maxMutations = maxMutations;\n\n    [this.#priorWindow, this.#nextWindow] = this.#newWindows();\n  }\n\n  canDo(): boolean {\n    const now = Date.now();\n\n    // If the current sliding window is completely past the next window, we need new windows.\n    if (\n      now - this.#windowSizeMs >\n      this.#nextWindow.start + this.#windowSizeMs\n    ) {\n      [this.#priorWindow, this.#nextWindow] = this.#newWindows();\n    }\n\n    // Has the current sliding window moved completely into the next window?\n    // Then rotate the windows.\n    if (now - this.#windowSizeMs >= this.#nextWindow.start) {\n      this.#rotateWindows();\n    }\n\n    // Now compute the total mutations in the current window.\n    // Weighted by the fraction of the window that has passed.\n    const totalCalls = this.totalCallsForTime(now);\n    const canDo = totalCalls < this.#maxMutations;\n\n    // We don't increment counts if we throttled the user. This is so the user can\n    // eventually do the action they are trying to do. If we incremented on every\n    // attempt they can end up in a case where their excessive retries lock them out\n    // continuously.\n    if (canDo) {\n      // If \"now\" in the prior or next window? Increment the count for the correct window.\n      if (now < this.#nextWindow.start) {\n        this.#priorWindow.count++;\n      } else {\n        this.#nextWindow.count++;\n      }\n    }\n\n    return canDo;\n  }\n\n  totalCallsForTime(now: number): number {\n    let fraction: number;\n    if (now < this.#priorWindow.start + this.#windowSizeMs) {\n      fraction = 1;\n    } else {\n      fraction =\n        (this.#priorWindow.start +\n          (this.#windowSizeMs - 1) -\n          (now - this.#windowSizeMs)) /\n        this.#windowSizeMs;\n    }\n    if (fraction < 0) {\n      fraction = 0;\n    }\n    assert(\n      fraction <= 1,\n      'The past cannot contribute more than the a full window.',\n    );\n    const totalCalls =\n      this.#priorWindow.count * fraction + this.#nextWindow.count;\n    return totalCalls;\n  }\n\n  get priorWindow(): Window {\n    return this.#priorWindow;\n  }\n\n  get nextWindow(): Window {\n    return this.#nextWindow;\n  }\n\n  /**\n   * If there were no mutations in over `windowSizeMs` we need new windows.\n   */\n  #newWindows() {\n    const now = Date.now();\n    const start = now - (now % this.#windowSizeMs);\n    return [\n      {\n        // clamp to a `windowSizeMs` boundary to start the prior window.\n        start,\n        count: 0,\n      },\n      {\n        start: start + this.#windowSizeMs,\n        count: 0,\n      },\n    ];\n  }\n\n  #rotateWindows() {\n    this.#priorWindow = this.#nextWindow;\n    this.#nextWindow = {\n      start: this.#priorWindow.start + this.#windowSizeMs,\n      count: 0,\n    };\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkDA,IAAa,uBAAb,MAAkC;CAChC;CACA;CAEA;CACA;CAEA,YAAY,cAAsB,cAAsB;EACtD,KAAK,gBAAgB;EACrB,KAAK,gBAAgB;EAErB,CAAC,KAAK,cAAc,KAAK,eAAe,KAAK,YAAY;CAC3D;CAEA,QAAiB;EACf,MAAM,MAAM,KAAK,IAAI;EAGrB,IACE,MAAM,KAAK,gBACX,KAAK,YAAY,QAAQ,KAAK,eAE9B,CAAC,KAAK,cAAc,KAAK,eAAe,KAAK,YAAY;EAK3D,IAAI,MAAM,KAAK,iBAAiB,KAAK,YAAY,OAC/C,KAAK,eAAe;EAMtB,MAAM,QADa,KAAK,kBAAkB,GAC5B,IAAa,KAAK;EAMhC,IAAI,OAAO;GAET,IAAI,MAAM,KAAK,YAAY,OACzB,KAAK,aAAa;QAElB,KAAK,YAAY;EAErB;EAEA,OAAO;CACT;CAEA,kBAAkB,KAAqB;EACrC,IAAI;EACJ,IAAI,MAAM,KAAK,aAAa,QAAQ,KAAK,eACvC,WAAW;OAEX,YACG,KAAK,aAAa,SAChB,KAAK,gBAAgB,MACrB,MAAM,KAAK,kBACd,KAAK;EAET,IAAI,WAAW,GACb,WAAW;EAEb,OACE,YAAY,GACZ,yDACF;EAGA,OADE,KAAK,aAAa,QAAQ,WAAW,KAAK,YAAY;CAE1D;CAEA,IAAI,cAAsB;EACxB,OAAO,KAAK;CACd;CAEA,IAAI,aAAqB;EACvB,OAAO,KAAK;CACd;;;;CAKA,cAAc;EACZ,MAAM,MAAM,KAAK,IAAI;EACrB,MAAM,QAAQ,MAAO,MAAM,KAAK;EAChC,OAAO,CACL;GAEE;GACA,OAAO;EACT,GACA;GACE,OAAO,QAAQ,KAAK;GACpB,OAAO;EACT,CACF;CACF;CAEA,iBAAiB;EACf,KAAK,eAAe,KAAK;EACzB,KAAK,cAAc;GACjB,OAAO,KAAK,aAAa,QAAQ,KAAK;GACtC,OAAO;EACT;CACF;AACF"}

package/out/zero-cache/src/services/litestream/commands.js

@@ -2,9 +2,9 @@ import { parse, valita_exports } from "../../../../shared/src/valita.js";
 import { must } from "../../../../shared/src/must.js";
 import "../../config/normalize.js";
 import { Database } from "../../../../zqlite/src/db.js";
-import { getSubscriptionState } from "../replicator/schema/replication-state.js";
-import { StatementRunner } from "../../db/statements.js";
 import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../db/sqlite-corruption.js";
+import { StatementRunner } from "../../db/statements.js";
+import { getSubscriptionState } from "../replicator/schema/replication-state.js";
 import { deleteLiteDB } from "../../db/delete-lite-db.js";
 import { deleteChangeLogDB } from "../replicator/change-log-db.js";
 import { litestreamSocketPath } from "./litestream-controller.js";
@@ -464,6 +464,6 @@ function parseBackupCreatedTimes(lc, command, output) {
 	return times;
 }
 //#endregion
-export { backupDestinationURL, deleteLitestreamMetaDir, getLastBackupTime, readLocalReplica, startReplicaBackupProcess, tryRestore };
+export { backupDestinationURL, deleteLitestreamMetaDir, getLastBackupTime, parseBackupCreatedTimes, readLocalReplica, startReplicaBackupProcess, tryRestore };
 
 //# sourceMappingURL=commands.js.map