@rocicorp/zero 1.11.0-canary.13 → 1.11.0-canary.14

package/out/zero-cache/src/services/change-streamer/subscriber.js

@@ -31,6 +31,10 @@ var Subscriber = class {
 	#backlogBackpressure;
 	#backlogFullWaiters = /* @__PURE__ */ new Set();
 	#onAck;
+	#backfills;
+	#onAligned;
+	#onBackfillIgnored;
+	#aligned = false;
 	constructor(protocolVersion, id, mode, watermark, downstream, latestStatus, options = {}) {
 		this.#protocolVersion = protocolVersion;
 		this.id = id;
@@ -43,6 +47,29 @@ var Subscriber = class {
 		this.#backlog = new RingBuffer();
 		this.#backlogBackpressure = new ByteBackpressureGate(options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES, options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO);
 		this.#onAck = options.onAck;
+		this.#backfills = options.backfills;
+		this.#onAligned = options.onAligned;
+		this.#onBackfillIgnored = options.onBackfillIgnored;
+	}
+	/**
+	* The subscriber's tracked pending backfills, or `undefined` if the
+	* subscriber does not report them (i.e. protocol < v8).
+	*/
+	get backfills() {
+		return this.#backfills;
+	}
+	/**
+	* Whether the subscriber has been sent all changes up to the head of the
+	* stream (see {@link SubscriberOptions.onAligned}).
+	*/
+	get aligned() {
+		return this.#aligned;
+	}
+	#trackBackfills(change) {
+		if (this.#backfills) {
+			const ignored = this.#backfills.applySerialized(change);
+			if (ignored.length && this.#aligned) this.#onBackfillIgnored?.(ignored);
+		}
 	}
 	get watermark() {
 		return this.#watermark;
@@ -106,6 +133,7 @@ var Subscriber = class {
 				break;
 			}
 			if (commitWatermark) this.#watermark = commitWatermark;
+			for (const change of changes) this.#trackBackfills(change);
 			return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);
 		}
 		const promises = [];
@@ -154,6 +182,7 @@ var Subscriber = class {
 		if (watermark <= this.watermark) return;
 		if (!this.supportsMessage(tag)) return;
 		if (tag === "commit") this.#watermark = watermark;
+		this.#trackBackfills(change);
 		const result = await this.#sendStringifiedDownstream(json);
 		if (tag === "commit" && result === "consumed") {
 			const acked = max(this.#acked, watermark);
@@ -342,9 +371,14 @@ var Subscriber = class {
 			for (;;) {
 				const change = this.#backlog?.shift();
 				if (!change) {
+					const closed = this.#backlog === null;
 					this.#backlog = null;
 					this.#backlogBytes = 0;
 					this.#backlogBackpressure.releaseIfUnderLowWater(this.#bufferedBacklogBytes);
+					if (!closed && !this.#aligned) {
+						this.#aligned = true;
+						this.#onAligned?.(this);
+					}
 					break;
 				}
 				const bytes = change[2].length;

package/out/zero-cache/src/services/change-streamer/subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#latestStatus","#wsBatched","#backlogBackpressure","#backlogFullWaiters","#onAck","#watermark","#acked","#backlog","#bufferedBacklogBytes","#pushBacklog","#maybeWaitForBacklogSpace","#sendChange","#initialized","#sendPreSerializedDownstream","#sendDownstream","#initialize","#backlogDrain","#drainBacklog","#sendStringifiedDownstream","#pending","#pendingBytes","#processed","#backlogCount","#samples","#totalBufferedBytes","#missedLastTimeout","#laggingSinceMs","#backlogBytes","#resolveBacklogFullWaiters","#backlogInFlightBytes","#lowWaterBytes","#waiters"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n  wsBatched?: boolean | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n};\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly #downstream: Subscription<string | PreSerializedBatch>;\n  readonly #latestStatus: () => Status;\n  readonly #wsBatched: boolean;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<WatermarkedChange> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string | PreSerializedBatch>,\n    latestStatus: () => Status,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.#downstream = downstream;\n    this.#latestStatus = latestStatus;\n    this.#wsBatched = options.wsBatched ?? false;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  sendBatch(\n    changes: readonly WatermarkedChange[],\n    preSerialized?: PreSerializedBatch | undefined,\n  ): Promise<void> {\n    if (changes.length === 0) {\n      return promiseVoid;\n    }\n\n    if (\n      this.#wsBatched &&\n      preSerialized &&\n      !this.#backlog &&\n      this.#initialized &&\n      changes[0][0] > this.#watermark &&\n      (this.#protocolVersion >= 5 ||\n        changes.every(c => this.supportsMessage(c[1])))\n    ) {\n      let commitWatermark: string | undefined;\n      for (let i = changes.length - 1; i >= 0; i--) {\n        if (changes[i][1] === 'commit') {\n          commitWatermark = changes[i][0];\n          break;\n        }\n      }\n      if (commitWatermark) {\n        this.#watermark = commitWatermark;\n      }\n      return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n    }\n\n    const promises: Promise<void>[] = [];\n    for (const change of changes) {\n      const p = this.send(change);\n      if (p !== promiseVoid) {\n        promises.push(p);\n      }\n    }\n    if (promises.length === 0) {\n      return promiseVoid;\n    }\n    if (promises.length === 1) {\n      return promises[0];\n    }\n    return Promise.all(promises).then(() => {});\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      this.sendStatus(this.#latestStatus());\n    }\n  }\n\n  sendStatus(status: Status) {\n    if (this.#initialized) {\n      void this.#sendDownstream(['status', status]);\n    }\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      // Sends can complete out of order (e.g. the bounded window in\n      // #drainBacklog), so the ack only advances monotonically, and listeners\n      // are only notified when it does.\n      const acked = max(this.#acked, watermark);\n      if (acked !== this.#acked) {\n        this.#acked = acked;\n        this.#onAck?.(acked);\n      }\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  async #sendPreSerializedDownstream(\n    batch: PreSerializedBatch,\n    commitWatermark?: string | undefined,\n  ): Promise<void> {\n    const size = batch.byteLength;\n    this.#pending += batch.changes.length;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(batch);\n    try {\n      const outcome = await result;\n      if (commitWatermark && outcome === 'consumed') {\n        const acked = max(this.#acked, commitWatermark);\n        if (acked !== this.#acked) {\n          this.#acked = acked;\n          this.#onAck?.(acked);\n        }\n      }\n    } finally {\n      this.#pending -= batch.changes.length;\n      this.#pendingBytes -= size;\n      this.#processed += batch.changes.length;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(error?: ErrorType, message?: string) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream(['error', {type: error, message}]).finally(() =>\n        this.#downstream.cancel(),\n      );\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(change: WatermarkedChange) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(change);\n    this.#backlogBytes += change[2].length;\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const change = this.#backlog?.shift();\n        if (!change) {\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          break;\n        }\n\n        const bytes = change[2].length;\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const promise = this.#sendChange(change).finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAqBA,IAAM,mCAAmC,KAAK,OAAO;AACrD,IAAM,kCAAkC;;;;;;;;AAoCxC,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CAEA,YACE,iBACA,IACA,MACA,WACA,YACA,cACA,UAA6B,CAAC,GAC9B;EACA,KAAKA,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAKC,cAAc;EACnB,KAAKC,gBAAgB;EACrB,KAAKC,aAAa,QAAQ,aAAa;EACvC,KAAKI,aAAa;EAClB,KAAKC,SAAS;EACd,KAAKC,WAAW,IAAI,WAAW;EAC/B,KAAKL,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAKE,SAAS,QAAQ;CACxB;CAEA,IAAI,YAAY;EACd,OAAO,KAAKC;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAKC;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAKE,yBAAyB,KAAKN,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAKK,aAAa,QAAQ,KAAKC,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAKL,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAKA,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAKE,YAAY;GAC/B,IAAI,KAAKE,UAAU;IAIjB,KAAKE,aAAa,MAAM;IACxB,OAAO,KAAKC,0BAA0B;GACxC;GACA,OAAO,KAAKC,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAKV,cACL,iBACA,CAAC,KAAKM,YACN,KAAKK,gBACL,QAAQ,GAAG,KAAK,KAAKP,eACpB,KAAKP,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,GAAG,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,GAAG;IAC7B;GACF;GAEF,IAAI,iBACF,KAAKO,aAAa;GAEpB,OAAO,KAAKQ,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,EAAE,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAKD,cAAc;GACtB,KAAKA,eAAe;GACpB,KAAK,WAAW,KAAKZ,cAAc,CAAC;EACtC;CACF;CAEA,WAAW,QAAgB;EACzB,IAAI,KAAKY,cACP,KAAUE,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAEhD;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAKC,YAAY;EACjB,MAAM,KAAKJ,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAKI,YAAY;EACjB,IAAI,CAAC,KAAKR,UACR,OAAO,KAAKS,iBAAiB;EAE/B,IAAI,CAAC,KAAKA,eAAe;GAKvB,KAAKA,gBAAgB,KAAKC,cAAc;GACxC,KAAUD,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAKA;CACd;CAEA,MAAML,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAKN,aAAa;EAEpB,MAAM,SAAS,MAAM,KAAKa,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YAAY;GAI7C,MAAM,QAAQ,IAAI,KAAKZ,QAAQ,SAAS;GACxC,IAAI,UAAU,KAAKA,QAAQ;IACzB,KAAKA,SAAS;IACd,KAAKF,SAAS,KAAK;GACrB;EACF;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAKc,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAMA,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAKC;EACL,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAKrB,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAKoB;GACL,KAAKC,iBAAiB;GACtB,KAAKC;EACP;CACF;CAEA,MAAMR,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAKM,YAAY,MAAM,QAAQ;EAC/B,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAKrB,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YAAY;IAC7C,MAAM,QAAQ,IAAI,KAAKO,QAAQ,eAAe;IAC9C,IAAI,UAAU,KAAKA,QAAQ;KACzB,KAAKA,SAAS;KACd,KAAKF,SAAS,KAAK;IACrB;GACF;EACF,UAAU;GACR,KAAKe,YAAY,MAAM,QAAQ;GAC/B,KAAKC,iBAAiB;GACtB,KAAKC,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAKF,WAAW,KAAKG;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAKD;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAKE,SAAS,UAAU,YAC7B,KAAKA,SAAS,MAAM;EAEtB,KAAKA,SAAS,KAAK;GAAC,WAAW,KAAKF;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAKE,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAKD;GACd,cAAc,KAAKd;GACnB,oBAAoB,KAAKgB;GACzB,mBAAmB,KAAKC;EAC1B;EAEF,MAAM,OAAO,KAAKF,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAKA,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAKD;GACd,cAAc,KAAKd;GACnB,oBAAoB,KAAKgB;GACzB,mBAAmB,KAAKC;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAKA,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAKC,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAKD,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAKC,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAKA,oBAAoB;EACzB,OAAO,MAAM,KAAKA;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAK5B,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MAAM,OAAmB,SAAkB;EAGzC,KAAKS,WAAW;EAChB,KAAKoB,gBAAgB;EACrB,KAAKzB,qBAAqB,WAAW;EAGrC,KAAK0B,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAUd,gBAAgB,CAAC,SAAS;GAAC,MAAM;GAAO;EAAO,CAAC,CAAC,EAAE,cAC3D,KAAKf,YAAY,OAAO,CAC1B;OAEA,KAAKA,YAAY,OAAO;CAE5B;CAEA,IAAIuB,gBAAgB;EAClB,OAAO,KAAKf,UAAU,QAAQ;CAChC;CAEA,IAAIC,wBAAwB;EAI1B,OAAO,KAAKmB,gBAAgB,KAAKE;CACnC;CAEA,IAAIL,sBAAsB;EACxB,OAAO,KAAKhB,wBAAwB,KAAKY;CAC3C;CAEA,aAAa,QAA2B;EACtC,OAAO,KAAKb,UAAU,gDAAgD;EACtE,KAAKA,SAAS,KAAK,MAAM;EACzB,KAAKoB,iBAAiB,OAAO,GAAG;EAChC,IAAI,KAAK,aACP,KAAKC,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAKzB,mBAAmB;EAC5C,KAAKA,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAKD,qBAAqB,aAAa,KAAKM,qBAAqB;CAC1E;CAEA,MAAMS,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,SAAS,KAAKV,UAAU,MAAM;IACpC,IAAI,CAAC,QAAQ;KACX,KAAKA,WAAW;KAChB,KAAKoB,gBAAgB;KACrB,KAAKzB,qBAAqB,uBACxB,KAAKM,qBACP;KACA;IACF;IAEA,MAAM,QAAQ,OAAO,GAAG;IACxB,KAAKmB,iBAAiB;IACtB,KAAKE,yBAAyB;IAC9B,KAAK3B,qBAAqB,uBACxB,KAAKM,qBACP;IAKA,MAAM,UAAU,KAAKG,YAAY,MAAM,EAAE,cAAc;KACrD,KAAKkB,yBAAyB;KAC9B,KAAK3B,qBAAqB,uBACxB,KAAKM,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAKN,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAKc,gBAAgB;GACrB,KAAKd,qBAAqB,uBACxB,KAAKM,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAKsB,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAKC,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAKA,SAAS,WAAW,KAAK,gBAAgB,KAAKD,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAKC,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF"}
+{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#latestStatus","#wsBatched","#backlogBackpressure","#backlogFullWaiters","#onAck","#backfills","#onAligned","#onBackfillIgnored","#watermark","#acked","#backlog","#aligned","#bufferedBacklogBytes","#pushBacklog","#maybeWaitForBacklogSpace","#sendChange","#initialized","#trackBackfills","#sendPreSerializedDownstream","#sendDownstream","#initialize","#backlogDrain","#drainBacklog","#sendStringifiedDownstream","#pending","#pendingBytes","#processed","#backlogCount","#samples","#totalBufferedBytes","#missedLastTimeout","#laggingSinceMs","#backlogBytes","#resolveBacklogFullWaiters","#backlogInFlightBytes","#lowWaterBytes","#waiters"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n  wsBatched?: boolean | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n\n  /**\n   * The subscriber's pending backfills (protocol v8+), which are tracked\n   * over the changes sent to the subscriber.\n   */\n  backfills?: BackfillState | undefined;\n\n  /**\n   * Called once the subscriber has been sent all changes up to the head of\n   * the stream, i.e. its catchup and backlog have been flushed. From this\n   * point on, the subscriber's tracked {@link backfills} reflect the same\n   * position in the stream as that of the change-streamer.\n   */\n  onAligned?: ((subscriber: Subscriber) => void) | undefined;\n\n  /**\n   * Called when a `backfill` or `backfill-completed` message is ignored for\n   * some of the subscriber's pending backfill columns, after the subscriber\n   * is aligned.\n   */\n  onBackfillIgnored?: ((columns: string[]) => void) | undefined;\n};\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly #downstream: Subscription<string | PreSerializedBatch>;\n  readonly #latestStatus: () => Status;\n  readonly #wsBatched: boolean;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<WatermarkedChange> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n  readonly #backfills: BackfillState | undefined;\n  readonly #onAligned: ((subscriber: Subscriber) => void) | undefined;\n  readonly #onBackfillIgnored: ((columns: string[]) => void) | undefined;\n  #aligned = false;\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string | PreSerializedBatch>,\n    latestStatus: () => Status,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.#downstream = downstream;\n    this.#latestStatus = latestStatus;\n    this.#wsBatched = options.wsBatched ?? false;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n    this.#backfills = options.backfills;\n    this.#onAligned = options.onAligned;\n    this.#onBackfillIgnored = options.onBackfillIgnored;\n  }\n\n  /**\n   * The subscriber's tracked pending backfills, or `undefined` if the\n   * subscriber does not report them (i.e. protocol < v8).\n   */\n  get backfills(): BackfillState | undefined {\n    return this.#backfills;\n  }\n\n  /**\n   * Whether the subscriber has been sent all changes up to the head of the\n   * stream (see {@link SubscriberOptions.onAligned}).\n   */\n  get aligned(): boolean {\n    return this.#aligned;\n  }\n\n  #trackBackfills(change: WatermarkedChange) {\n    if (this.#backfills) {\n      const ignored = this.#backfills.applySerialized(change);\n      if (ignored.length && this.#aligned) {\n        this.#onBackfillIgnored?.(ignored);\n      }\n    }\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  sendBatch(\n    changes: readonly WatermarkedChange[],\n    preSerialized?: PreSerializedBatch | undefined,\n  ): Promise<void> {\n    if (changes.length === 0) {\n      return promiseVoid;\n    }\n\n    if (\n      this.#wsBatched &&\n      preSerialized &&\n      !this.#backlog &&\n      this.#initialized &&\n      changes[0][0] > this.#watermark &&\n      (this.#protocolVersion >= 5 ||\n        changes.every(c => this.supportsMessage(c[1])))\n    ) {\n      let commitWatermark: string | undefined;\n      for (let i = changes.length - 1; i >= 0; i--) {\n        if (changes[i][1] === 'commit') {\n          commitWatermark = changes[i][0];\n          break;\n        }\n      }\n      if (commitWatermark) {\n        this.#watermark = commitWatermark;\n      }\n      for (const change of changes) {\n        this.#trackBackfills(change);\n      }\n      return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n    }\n\n    const promises: Promise<void>[] = [];\n    for (const change of changes) {\n      const p = this.send(change);\n      if (p !== promiseVoid) {\n        promises.push(p);\n      }\n    }\n    if (promises.length === 0) {\n      return promiseVoid;\n    }\n    if (promises.length === 1) {\n      return promises[0];\n    }\n    return Promise.all(promises).then(() => {});\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      this.sendStatus(this.#latestStatus());\n    }\n  }\n\n  sendStatus(status: Status) {\n    if (this.#initialized) {\n      void this.#sendDownstream(['status', status]);\n    }\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    this.#trackBackfills(change);\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      // Sends can complete out of order (e.g. the bounded window in\n      // #drainBacklog), so the ack only advances monotonically, and listeners\n      // are only notified when it does.\n      const acked = max(this.#acked, watermark);\n      if (acked !== this.#acked) {\n        this.#acked = acked;\n        this.#onAck?.(acked);\n      }\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  async #sendPreSerializedDownstream(\n    batch: PreSerializedBatch,\n    commitWatermark?: string | undefined,\n  ): Promise<void> {\n    const size = batch.byteLength;\n    this.#pending += batch.changes.length;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(batch);\n    try {\n      const outcome = await result;\n      if (commitWatermark && outcome === 'consumed') {\n        const acked = max(this.#acked, commitWatermark);\n        if (acked !== this.#acked) {\n          this.#acked = acked;\n          this.#onAck?.(acked);\n        }\n      }\n    } finally {\n      this.#pending -= batch.changes.length;\n      this.#pendingBytes -= size;\n      this.#processed += batch.changes.length;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(error?: ErrorType, message?: string) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream(['error', {type: error, message}]).finally(() =>\n        this.#downstream.cancel(),\n      );\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(change: WatermarkedChange) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(change);\n    this.#backlogBytes += change[2].length;\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const change = this.#backlog?.shift();\n        if (!change) {\n          const closed = this.#backlog === null;\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          if (!closed && !this.#aligned) {\n            // All changes up to the head have been handed to #sendChange()\n            // (and thus tracked), and subsequent changes are sent directly.\n            this.#aligned = true;\n            this.#onAligned?.(this);\n          }\n          break;\n        }\n\n        const bytes = change[2].length;\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const promise = this.#sendChange(change).finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAsBA,IAAM,mCAAmC,KAAK,OAAO;AACrD,IAAM,kCAAkC;;;;;;;;AAyDxC,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CACA;CACA;CACA;CACA,WAAW;CAEX,YACE,iBACA,IACA,MACA,WACA,YACA,cACA,UAA6B,CAAC,GAC9B;EACA,KAAKA,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAKC,cAAc;EACnB,KAAKC,gBAAgB;EACrB,KAAKC,aAAa,QAAQ,aAAa;EACvC,KAAKO,aAAa;EAClB,KAAKC,SAAS;EACd,KAAKC,WAAW,IAAI,WAAW;EAC/B,KAAKR,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAKE,SAAS,QAAQ;EACtB,KAAKC,aAAa,QAAQ;EAC1B,KAAKC,aAAa,QAAQ;EAC1B,KAAKC,qBAAqB,QAAQ;CACpC;;;;;CAMA,IAAI,YAAuC;EACzC,OAAO,KAAKF;CACd;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAKM;CACd;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAKN,YAAY;GACnB,MAAM,UAAU,KAAKA,WAAW,gBAAgB,MAAM;GACtD,IAAI,QAAQ,UAAU,KAAKM,UACzB,KAAKJ,qBAAqB,OAAO;EAErC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAKC;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAKC;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAKG,yBAAyB,KAAKV,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAKQ,aAAa,QAAQ,KAAKE,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAKT,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAKA,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAKK,YAAY;GAC/B,IAAI,KAAKE,UAAU;IAIjB,KAAKG,aAAa,MAAM;IACxB,OAAO,KAAKC,0BAA0B;GACxC;GACA,OAAO,KAAKC,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAKd,cACL,iBACA,CAAC,KAAKS,YACN,KAAKM,gBACL,QAAQ,GAAG,KAAK,KAAKR,eACpB,KAAKV,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,GAAG,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,GAAG;IAC7B;GACF;GAEF,IAAI,iBACF,KAAKU,aAAa;GAEpB,KAAK,MAAM,UAAU,SACnB,KAAKS,gBAAgB,MAAM;GAE7B,OAAO,KAAKC,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,EAAE,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAKF,cAAc;GACtB,KAAKA,eAAe;GACpB,KAAK,WAAW,KAAKhB,cAAc,CAAC;EACtC;CACF;CAEA,WAAW,QAAgB;EACzB,IAAI,KAAKgB,cACP,KAAUG,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAEhD;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAKC,YAAY;EACjB,MAAM,KAAKL,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAKK,YAAY;EACjB,IAAI,CAAC,KAAKV,UACR,OAAO,KAAKW,iBAAiB;EAE/B,IAAI,CAAC,KAAKA,eAAe;GAKvB,KAAKA,gBAAgB,KAAKC,cAAc;GACxC,KAAUD,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAKA;CACd;CAEA,MAAMN,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAKP,aAAa;EAEpB,KAAKS,gBAAgB,MAAM;EAC3B,MAAM,SAAS,MAAM,KAAKM,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YAAY;GAI7C,MAAM,QAAQ,IAAI,KAAKd,QAAQ,SAAS;GACxC,IAAI,UAAU,KAAKA,QAAQ;IACzB,KAAKA,SAAS;IACd,KAAKL,SAAS,KAAK;GACrB;EACF;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAKmB,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAMA,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAKC;EACL,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK1B,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAKyB;GACL,KAAKC,iBAAiB;GACtB,KAAKC;EACP;CACF;CAEA,MAAMR,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAKM,YAAY,MAAM,QAAQ;EAC/B,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK1B,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YAAY;IAC7C,MAAM,QAAQ,IAAI,KAAKU,QAAQ,eAAe;IAC9C,IAAI,UAAU,KAAKA,QAAQ;KACzB,KAAKA,SAAS;KACd,KAAKL,SAAS,KAAK;IACrB;GACF;EACF,UAAU;GACR,KAAKoB,YAAY,MAAM,QAAQ;GAC/B,KAAKC,iBAAiB;GACtB,KAAKC,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAKF,WAAW,KAAKG;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAKD;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAKE,SAAS,UAAU,YAC7B,KAAKA,SAAS,MAAM;EAEtB,KAAKA,SAAS,KAAK;GAAC,WAAW,KAAKF;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAKE,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAKD;GACd,cAAc,KAAKf;GACnB,oBAAoB,KAAKiB;GACzB,mBAAmB,KAAKC;EAC1B;EAEF,MAAM,OAAO,KAAKF,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAKA,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAKD;GACd,cAAc,KAAKf;GACnB,oBAAoB,KAAKiB;GACzB,mBAAmB,KAAKC;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAKA,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAKC,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAKD,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAKC,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAKA,oBAAoB;EACzB,OAAO,MAAM,KAAKA;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAKjC,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MAAM,OAAmB,SAAkB;EAGzC,KAAKY,WAAW;EAChB,KAAKsB,gBAAgB;EACrB,KAAK9B,qBAAqB,WAAW;EAGrC,KAAK+B,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAUd,gBAAgB,CAAC,SAAS;GAAC,MAAM;GAAO;EAAO,CAAC,CAAC,EAAE,cAC3D,KAAKpB,YAAY,OAAO,CAC1B;OAEA,KAAKA,YAAY,OAAO;CAE5B;CAEA,IAAI4B,gBAAgB;EAClB,OAAO,KAAKjB,UAAU,QAAQ;CAChC;CAEA,IAAIE,wBAAwB;EAI1B,OAAO,KAAKoB,gBAAgB,KAAKE;CACnC;CAEA,IAAIL,sBAAsB;EACxB,OAAO,KAAKjB,wBAAwB,KAAKa;CAC3C;CAEA,aAAa,QAA2B;EACtC,OAAO,KAAKf,UAAU,gDAAgD;EACtE,KAAKA,SAAS,KAAK,MAAM;EACzB,KAAKsB,iBAAiB,OAAO,GAAG;EAChC,IAAI,KAAK,aACP,KAAKC,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAK9B,mBAAmB;EAC5C,KAAKA,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAKD,qBAAqB,aAAa,KAAKU,qBAAqB;CAC1E;CAEA,MAAMU,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,SAAS,KAAKZ,UAAU,MAAM;IACpC,IAAI,CAAC,QAAQ;KACX,MAAM,SAAS,KAAKA,aAAa;KACjC,KAAKA,WAAW;KAChB,KAAKsB,gBAAgB;KACrB,KAAK9B,qBAAqB,uBACxB,KAAKU,qBACP;KACA,IAAI,CAAC,UAAU,CAAC,KAAKD,UAAU;MAG7B,KAAKA,WAAW;MAChB,KAAKL,aAAa,IAAI;KACxB;KACA;IACF;IAEA,MAAM,QAAQ,OAAO,GAAG;IACxB,KAAK0B,iBAAiB;IACtB,KAAKE,yBAAyB;IAC9B,KAAKhC,qBAAqB,uBACxB,KAAKU,qBACP;IAKA,MAAM,UAAU,KAAKG,YAAY,MAAM,EAAE,cAAc;KACrD,KAAKmB,yBAAyB;KAC9B,KAAKhC,qBAAqB,uBACxB,KAAKU,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAKV,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAKmB,gBAAgB;GACrB,KAAKnB,qBAAqB,uBACxB,KAAKU,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAKuB,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAKC,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAKA,SAAS,WAAW,KAAK,gBAAgB,KAAKD,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAKC,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF"}

package/out/zero-cache/src/services/plannerz.d.ts

@@ -0,0 +1,159 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import { Database } from '../../../zqlite/src/db.ts';
+import { type SampleValueKind } from '../../../zqlite/src/sqlite-stat4-sample.ts';
+import type { NormalizedZeroConfig as ZeroConfig } from '../config/normalize.ts';
+/**
+ * A fixed glossary, sent with every bundle. The audience is a language model
+ * that has the user's codebase but has never seen these statistics, and would
+ * otherwise read a sampled estimate as an exact count.
+ */
+declare const ABOUT: {
+    readonly whatThisIs: string;
+    readonly stat1Format: string;
+    readonly stat4Format: string;
+    readonly howThePlannerUsesThis: readonly [string, string, string];
+    readonly whatYouCanChange: readonly [string, 'A `limit`, which lets the planner stop early.', string, 'The planner flags in `server.planner`.'];
+    readonly caveats: readonly [string, string, string];
+    readonly nextStep: string;
+};
+export type Stat1 = {
+    /** The raw `sqlite_stat1.stat` text, in case a flag is not parsed here. */
+    raw: string;
+    /** Estimated rows in the table. */
+    rows: number;
+    /**
+     * Average rows per distinct key prefix: the first entry is for a 1-column
+     * prefix, the second for 2 columns, and so on.
+     */
+    avgRowsPerPrefix: number[];
+    /** Non-numeric tokens, e.g. `unordered`, `noskipscan`, `sz=N`. */
+    flags: string[];
+};
+/** A `sqlite_stat4` sample with its values replaced by their types. */
+export type Stat4Sample = {
+    nEq: number[];
+    nLt: number[];
+    nDLt: number[];
+    /** One entry per key column, plus the trailing rowid. */
+    sample: SampleValueKind[];
+};
+/**
+ * A digest of an index's `sqlite_stat4` histogram, one entry per key prefix
+ * depth. This is what the default response carries: a full histogram runs to
+ * hundreds of samples per index, which is a lot of context to spend on data an
+ * LLM would only summarize anyway.
+ */
+export type Stat4PrefixSummary = {
+    /** Rows equal to the most common sampled key at this depth. */
+    maxRowsPerKey: number;
+    /** Median rows per sampled key at this depth. */
+    medianRowsPerKey: number;
+    /**
+     * Lower bound on distinct keys at this depth, from the last sample's count of
+     * distinct values less than it.
+     */
+    estimatedDistinctKeys: number;
+};
+export type Stat4Summary = {
+    samples: number;
+    /** The type of each key column, and of the trailing rowid. */
+    sampleKinds: SampleValueKind[];
+    /** One entry per key prefix depth: 1 column, then 2, and so on. */
+    perPrefix: Stat4PrefixSummary[];
+};
+export type IndexStats = {
+    name: string;
+    columns: {
+        name: string;
+        dir: 'ASC' | 'DESC';
+    }[];
+    unique: boolean;
+    partial: boolean;
+    stat1?: Stat1 | undefined;
+    stat4?: Stat4Summary | undefined;
+    /** The full histogram, only with `?stat4=full`. */
+    stat4Samples?: Stat4Sample[] | undefined;
+};
+export type ColumnStats = {
+    name: string;
+    dataType: string;
+    nullable: boolean;
+    /** The ZQL type, or `null` for a column clients cannot see. */
+    zqlType: string | null;
+    backfilling?: true | undefined;
+};
+export type TableStats = {
+    name: string;
+    columns: ColumnStats[];
+    primaryKey: string[] | undefined;
+    estimatedRows: number | undefined;
+    syncable: boolean;
+    indexes: IndexStats[];
+};
+export type PlannerzBundle = {
+    about: typeof ABOUT;
+    server: {
+        zeroVersion: string;
+        sqliteVersion: string;
+        replicaWatermark: string | undefined;
+        generatedAt: string;
+        planner: {
+            enableQueryPlanner: boolean;
+            enableCorrelatedPredicatePushdown: boolean;
+            enablePlannerAwarePushdown: boolean;
+        };
+    };
+    statsQuality: {
+        stat1Present: boolean;
+        stat4Rows: number;
+        method: string;
+        tablesWithoutStats: string[];
+    };
+    tables: TableStats[];
+};
+/**
+ * Parses a `sqlite_stat1.stat` value.
+ *
+ * @visibleForTesting
+ */
+export declare function parseStat1(raw: string): Stat1;
+/**
+ * Builds the `/plannerz` bundle from the replica's catalog, `sqlite_stat*`
+ * tables and the server config. No user table is read, so the cost does not
+ * depend on how much data the replica holds.
+ *
+ * @visibleForTesting
+ */
+export declare function buildPlannerzBundle(lc: LogContext, config: ZeroConfig, opts?: BundleOptions): PlannerzBundle;
+export type BundleOptions = {
+    /** Include the full `sqlite_stat4` histogram, not just the digest. */
+    fullStat4?: boolean | undefined;
+};
+/** @visibleForTesting */
+export declare function buildPlannerzBundleFromDB(lc: LogContext, db: Database, config: ZeroConfig, opts?: BundleOptions): PlannerzBundle;
+/**
+ * Digests an index's stat4 samples.
+ *
+ * Note that `nLt` and `nDLt` are stored as text, so SQL cannot order the
+ * samples by them numerically. This works off the whole set rather than
+ * relying on the samples' order: the highest `nDLt` is a lower bound on the
+ * distinct keys, since the sample holding it has that many distinct keys below
+ * it plus itself.
+ *
+ * @visibleForTesting
+ */
+export declare function summarizeStat4(samples: Stat4Sample[]): Stat4Summary;
+/**
+ * Serves the replica's schema, indexes and SQLite statistics for consumption by
+ * a language model, so that it can advise on query rewrites and indexes.
+ *
+ * Row data is never returned. See `designs/003_replica_stats_endpoint.md`.
+ *
+ * HTTP query parameters:
+ * * `pretty`: indents the JSON
+ * * `stat4=full`: includes the full sqlite_stat4 histogram, which is large
+ */
+export declare function handlePlannerzRequest(lc: LogContext, config: ZeroConfig, req: FastifyRequest, res: FastifyReply): Promise<void>;
+export {};
+//# sourceMappingURL=plannerz.d.ts.map

package/out/zero-cache/src/services/plannerz.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"plannerz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/plannerz.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAC1D,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAC;AACnD,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,4CAA4C,CAAC;AACpD,OAAO,KAAK,EAAC,oBAAoB,IAAI,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAuB/E;;;;GAIG;AACH,QAAA,MAAM,KAAK;aACT,UAAU;aAIV,WAAW;aAMX,WAAW;aAOX,qBAAqB;aAUrB,gBAAgB,oBAGd,+CAA+C,UAG/C,wCAAwC;aAE1C,OAAO;aASP,QAAQ;CAGA,CAAC;AAEX,MAAM,MAAM,KAAK,GAAG;IAClB,2EAA2E;IAC3E,GAAG,EAAE,MAAM,CAAC;IACZ,mCAAmC;IACnC,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,kEAAkE;IAClE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAEF,uEAAuE;AACvE,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,GAAG,EAAE,MAAM,EAAE,CAAC;IACd,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,yDAAyD;IACzD,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,+DAA+D;IAC/D,aAAa,EAAE,MAAM,CAAC;IACtB,iDAAiD;IACjD,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,WAAW,EAAE,eAAe,EAAE,CAAC;IAC/B,mEAAmE;IACnE,SAAS,EAAE,kBAAkB,EAAE,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,KAAK,GAAG,MAAM,CAAA;KAAC,EAAE,CAAC;IAC/C,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;IAC1B,KAAK,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACjC,mDAAmD;IACnD,YAAY,CAAC,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;CAC1C,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;IAClB,+DAA+D;IAC/D,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,WAAW,CAAC,EAAE,IAAI,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,OAAO,KAAK,CAAC;IACpB,MAAM,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;QACrC,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE;YACP,kBAAkB,EAAE,OAAO,CAAC;YAC5B,iCAAiC,EAAE,OAAO,CAAC;YAC3C,0BAA0B,EAAE,OAAO,CAAC;SACrC,CAAC;KACH,CAAC;IACF,YAAY,EAAE;QACZ,YAAY,EAAE,OAAO,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,kBAAkB,EAAE,MAAM,EAAE,CAAC;KAC9B,CAAC;IACF,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,KAAK,CAmB7C;AAmCD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAGhB;AAED,MAAM,MAAM,aAAa,GAAG;IAC1B,sEAAsE;IACtE,SAAS,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACjC,CAAC;AAEF,yBAAyB;AACzB,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,MAAM,EAAE,UAAU,EAClB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAsHhB;AAiBD;;;;;;;;;;GAUG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,YAAY,CAuBnE;AAmBD;;;;;;;;;GASG;AACH,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,iBA0BlB"}

package/out/zero-cache/src/services/plannerz.js

@@ -0,0 +1,263 @@
+import { getServerVersion, isAdminPasswordValid } from "../config/zero-config.js";
+import { Database } from "../../../zqlite/src/db.js";
+import { computeZqlSpecsFromLiteSpecs, listIndexes, listTables } from "../db/lite-tables.js";
+import { decodeSampleKinds } from "../../../zqlite/src/sqlite-stat4-sample.js";
+import { getReplicationState } from "./replicator/schema/replication-state.js";
+import { StatementRunner } from "../db/statements.js";
+import { _usingCtx } from "../../../_virtual/_@oxc-project_runtime@0.130.0/helpers/usingCtx.js";
+import auth from "basic-auth";
+//#region ../zero-cache/src/services/plannerz.ts
+/**
+* How the statistics in the replica were gathered. zero-cache only ever runs
+* `PRAGMA optimize`, and always under an `analysis_limit`, which samples rather
+* than scanning. A replica that someone ran `ANALYZE` on by hand has better
+* statistics, which is visible in `sqlite_stat4` having rows at all.
+*/
+var APPROXIMATE_METHOD = "PRAGMA optimize under an analysis_limit. Row counts and per-key averages are sampled estimates, and sqlite_stat4 is not collected.";
+var FULL_METHOD = "ANALYZE, with no analysis limit. sqlite_stat4 has rows, so a full ANALYZE ran against this replica at some point.";
+/**
+* A fixed glossary, sent with every bundle. The audience is a language model
+* that has the user's codebase but has never seen these statistics, and would
+* otherwise read a sampled estimate as an exact count.
+*/
+var ABOUT = {
+	whatThisIs: "Schema, indexes and SQLite statistics for the SQLite replica that zero-cache queries. Use it to decide how to rewrite ZQL queries and which indexes to add. It contains no row data.",
+	stat1Format: "sqlite_stat1.stat is \"<rows> <avg rows per distinct 1-column prefix> <... per 2-column prefix> ...\", parsed here into `rows` and `avgRowsPerPrefix`. NULLs count as ordinary values, so a sparse column looks less selective than it is. Trailing flags such as \"unordered\", \"noskipscan\" and \"sz=N\" are kept in `flags`.",
+	stat4Format: "sqlite_stat4 is a per-index histogram. For each sample, `nEq` is the rows equal to it, `nLt` the rows less than it, and `nDLt` the distinct values less than it, one entry per key column. A large `nEq` next to a small `nDLt` means a skewed column, where the average in stat1 hides the common values. Sample values are never returned: `sample` reports only each key column’s type.",
+	howThePlannerUsesThis: [
+		"Cost comes from SQLite scanstatus estimates for the SQL each source connection would run, so the statistics here are what the planner sees.",
+		"Join fanout comes from sqlite_stat4, else sqlite_stat1, else a default of 3. With no stat4 (the usual case) fanout on a nullable foreign key is overestimated, because stat1 counts NULLs.",
+		"The planner chooses per join whether to run it as a semi-join or to flip it so the child drives. An index that matches the child’s constraint plus ordering is what makes a flip cheap."
+	],
+	whatYouCanChange: [
+		"The ZQL query: `whereExists` vs `related`, the order of conditions, and ordering that matches an existing index.",
+		"A `limit`, which lets the planner stop early.",
+		"Indexes: add them in Postgres, not on the replica. The replica copies upstream indexes, so a new index reaches it through replication.",
+		"The planner flags in `server.planner`."
+	],
+	caveats: [
+		"Row counts are estimates from the statistics, not COUNT(*). This endpoint never scans a table.",
+		"Statistics are as of the last time they were gathered, which SQLite does not record. A table whose size changed a lot since then can be far off.",
+		"Tables in `tables` with `syncable: false` cannot be queried by clients: they lack a primary key or unique index, or every column has a type ZQL does not support."
+	],
+	nextStep: "POST an AST to /plannerz/analyze to see the plan the planner picks for one query, with the SQLite query plans it generates."
+};
+/**
+* Parses a `sqlite_stat1.stat` value.
+*
+* @visibleForTesting
+*/
+function parseStat1(raw) {
+	const tokens = raw.trim().split(SEPARATOR).filter(Boolean);
+	const numbers = [];
+	const flags = [];
+	for (const token of tokens) if (flags.length === 0 && COUNT.test(token)) numbers.push(parseInt(token, 10));
+	else flags.push(token);
+	return {
+		raw,
+		rows: numbers[0] ?? 0,
+		avgRowsPerPrefix: numbers.slice(1),
+		flags
+	};
+}
+/** Whitespace between the counts in a `sqlite_stat1` or `sqlite_stat4` value. */
+var SEPARATOR = /\s+/;
+/** A count, as opposed to a trailing flag such as `unordered` or `sz=N`. */
+var COUNT = /^\d+$/;
+function tableExists(db, name) {
+	return db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`).all(name).length > 0;
+}
+function parseCounts(raw) {
+	return raw.trim().split(SEPARATOR).filter(Boolean).map((n) => parseInt(n, 10));
+}
+/**
+* Builds the `/plannerz` bundle from the replica's catalog, `sqlite_stat*`
+* tables and the server config. No user table is read, so the cost does not
+* depend on how much data the replica holds.
+*
+* @visibleForTesting
+*/
+function buildPlannerzBundle(lc, config, opts = {}) {
+	try {
+		var _usingCtx$1 = _usingCtx();
+		return buildPlannerzBundleFromDB(lc, _usingCtx$1.u(new Database(lc, config.replica.file, { readonly: true })), config, opts);
+	} catch (_) {
+		_usingCtx$1.e = _;
+	} finally {
+		_usingCtx$1.d();
+	}
+}
+/** @visibleForTesting */
+function buildPlannerzBundleFromDB(lc, db, config, opts = {}) {
+	const tables = listReplicaTables(lc, db);
+	const indexes = listIndexes(db);
+	const zqlSpecs = computeZqlSpecsFromLiteSpecs(tables, indexes, { includeBackfillingColumns: false }, void 0, void 0, lc);
+	const stat1Present = tableExists(db, "sqlite_stat1");
+	const stat1ByTableAndIndex = /* @__PURE__ */ new Map();
+	const statsByTable = /* @__PURE__ */ new Map();
+	if (stat1Present) for (const { tbl, idx, stat } of db.prepare(`SELECT tbl, idx, stat FROM sqlite_stat1`).all()) {
+		const parsed = parseStat1(stat);
+		if (idx !== null) stat1ByTableAndIndex.set(`${tbl}.${idx}`, parsed);
+		if (idx === null || !statsByTable.has(tbl)) statsByTable.set(tbl, parsed);
+	}
+	const stat4Present = tableExists(db, "sqlite_stat4");
+	const stat4ByTableAndIndex = /* @__PURE__ */ new Map();
+	let stat4Rows = 0;
+	if (stat4Present) for (const { tbl, idx, neq, nlt, ndlt, sample } of db.prepare(`SELECT tbl, idx, neq, nlt, ndlt, sample FROM sqlite_stat4`).all()) {
+		stat4Rows++;
+		const samples = stat4ByTableAndIndex.get(`${tbl}.${idx}`) ?? [];
+		samples.push({
+			nEq: parseCounts(neq),
+			nLt: parseCounts(nlt),
+			nDLt: parseCounts(ndlt),
+			sample: decodeSampleKinds(sample)
+		});
+		stat4ByTableAndIndex.set(`${tbl}.${idx}`, samples);
+	}
+	const indexesByTable = /* @__PURE__ */ new Map();
+	for (const index of indexes) {
+		const key = `${index.tableName}.${index.name}`;
+		const stat4 = stat4ByTableAndIndex.get(key);
+		const forTable = indexesByTable.get(index.tableName) ?? [];
+		forTable.push({
+			name: index.name,
+			columns: Object.entries(index.columns).map(([name, dir]) => ({
+				name,
+				dir
+			})),
+			unique: index.unique,
+			partial: index.partial === true,
+			stat1: stat1ByTableAndIndex.get(key),
+			stat4: stat4?.length ? summarizeStat4(stat4) : void 0,
+			stat4Samples: opts.fullStat4 && stat4?.length ? stat4 : void 0
+		});
+		indexesByTable.set(index.tableName, forTable);
+	}
+	const tablesWithoutStats = [];
+	const tableStats = tables.map((table) => {
+		const zqlSpec = zqlSpecs.get(table.name)?.zqlSpec;
+		if (!statsByTable.has(table.name)) tablesWithoutStats.push(table.name);
+		const backfilling = new Set(table.backfilling ?? []);
+		return {
+			name: table.name,
+			columns: Object.entries(table.columns).map(([name, spec]) => ({
+				name,
+				dataType: spec.dataType,
+				nullable: !spec.notNull,
+				zqlType: zqlSpec?.[name]?.type ?? null,
+				...backfilling.has(name) ? { backfilling: true } : {}
+			})),
+			primaryKey: table.primaryKey ? [...table.primaryKey] : void 0,
+			estimatedRows: statsByTable.get(table.name)?.rows,
+			syncable: zqlSpec !== void 0,
+			indexes: indexesByTable.get(table.name) ?? []
+		};
+	});
+	return {
+		about: ABOUT,
+		server: {
+			zeroVersion: getServerVersion(config),
+			sqliteVersion: sqliteVersion(db),
+			replicaWatermark: replicaWatermark(lc, db),
+			generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+			planner: {
+				enableQueryPlanner: config.enableQueryPlanner,
+				enableCorrelatedPredicatePushdown: config.enableCorrelatedPredicatePushdown !== false,
+				enablePlannerAwarePushdown: config.enablePlannerAwarePushdown !== false
+			}
+		},
+		statsQuality: {
+			stat1Present,
+			stat4Rows,
+			method: stat4Rows > 0 ? FULL_METHOD : APPROXIMATE_METHOD,
+			tablesWithoutStats
+		},
+		tables: tableStats
+	};
+}
+/**
+* Lists the replica's tables, tolerating a replica that has no metadata tables
+* yet. `listTables` reads `_zero.tableMetadata` for backfill status, which does
+* not exist until initial sync has created it. Reporting the schema without
+* backfill status beats failing the whole request.
+*/
+function listReplicaTables(lc, db) {
+	try {
+		return listTables(db);
+	} catch (e) {
+		lc.debug?.("reading table metadata failed; listing tables without it", e);
+		return listTables(db, true, false);
+	}
+}
+/**
+* Digests an index's stat4 samples.
+*
+* Note that `nLt` and `nDLt` are stored as text, so SQL cannot order the
+* samples by them numerically. This works off the whole set rather than
+* relying on the samples' order: the highest `nDLt` is a lower bound on the
+* distinct keys, since the sample holding it has that many distinct keys below
+* it plus itself.
+*
+* @visibleForTesting
+*/
+function summarizeStat4(samples) {
+	const depth = Math.max(...samples.map((s) => s.nEq.length));
+	const perPrefix = [];
+	for (let d = 0; d < depth; d++) {
+		const counts = samples.map((s) => s.nEq[d]).filter((n) => n !== void 0).sort((a, b) => a - b);
+		const maxDistinctBelow = Math.max(...samples.map((s) => s.nDLt[d]).filter((n) => n !== void 0));
+		perPrefix.push({
+			maxRowsPerKey: counts.at(-1),
+			medianRowsPerKey: counts[Math.floor((counts.length - 1) / 2)],
+			estimatedDistinctKeys: maxDistinctBelow + 1
+		});
+	}
+	return {
+		samples: samples.length,
+		sampleKinds: samples[0].sample,
+		perPrefix
+	};
+}
+function sqliteVersion(db) {
+	const [{ version }] = db.prepare(`SELECT sqlite_version() AS version`).all();
+	return version;
+}
+function replicaWatermark(lc, db) {
+	try {
+		return getReplicationState(new StatementRunner(db)).stateVersion;
+	} catch (e) {
+		lc.debug?.("could not read replication state", e);
+		return;
+	}
+}
+/**
+* Serves the replica's schema, indexes and SQLite statistics for consumption by
+* a language model, so that it can advise on query rewrites and indexes.
+*
+* Row data is never returned. See `designs/003_replica_stats_endpoint.md`.
+*
+* HTTP query parameters:
+* * `pretty`: indents the JSON
+* * `stat4=full`: includes the full sqlite_stat4 histogram, which is large
+*/
+async function handlePlannerzRequest(lc, config, req, res) {
+	if (!isAdminPasswordValid(lc, config, auth(req)?.pass)) {
+		res.code(401).header("WWW-Authenticate", "Basic realm=\"Plannerz Protected Area\"").send("Unauthorized");
+		return;
+	}
+	const query = req.query;
+	const indent = query.pretty !== void 0 ? 2 : void 0;
+	let bundle;
+	try {
+		bundle = buildPlannerzBundle(lc, config, { fullStat4: query.stat4 === "full" });
+	} catch (e) {
+		lc.error?.("error building plannerz bundle", e);
+		await res.code(500).send({ error: String(e) });
+		return;
+	}
+	await res.header("Content-Type", "application/json").send(JSON.stringify(bundle, null, indent));
+}
+//#endregion
+export { handlePlannerzRequest };
+
+//# sourceMappingURL=plannerz.js.map