@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.10

package/out/zero-cache/src/services/change-streamer/subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#latestStatus","#backlogBackpressure","#watermark","#acked","#backlog","#pushBacklog","#maybeWaitForBacklogSpace","#sendChange","#initialized","#sendDownstream","#initialize","#backlogDrain","#drainBacklog","#sendStringifiedDownstream","#pending","#processed","#backlogCount","#samples","#bufferedBacklogBytes","#backlogBytes","#backlogInFlightBytes","#lowWaterBytes","#waiters"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ChangeTag, WatermarkedChange} from './change-streamer-service.ts';\nimport {type Downstream, type Status} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly #downstream: Subscription<string>;\n  readonly #latestStatus: () => Status;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<WatermarkedChange> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    watermark: string,\n    downstream: Subscription<string>,\n    latestStatus: () => Status,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.#downstream = downstream;\n    this.#latestStatus = latestStatus;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      this.sendStatus(this.#latestStatus());\n    }\n  }\n\n  sendStatus(status: Status) {\n    if (this.#protocolVersion >= 2 && this.#initialized) {\n      void this.#sendDownstream(['status', status]);\n    }\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      this.#acked = max(this.#acked, watermark);\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    this.#pending++;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#processed++;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): {\n    processRate: number;\n    pending: number;\n    backlog: number;\n    backlogBytes: number;\n  } {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n    };\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  fail(err?: unknown) {\n    this.close(ErrorType.Unknown, String(err));\n  }\n\n  close(error?: ErrorType, message?: string) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n\n    if (error) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream(['error', {type: error, message}]).finally(() =>\n        this.#downstream.cancel(),\n      );\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  #pushBacklog(change: WatermarkedChange) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(change);\n    this.#backlogBytes += change[2].length;\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const change = this.#backlog?.shift();\n        if (!change) {\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          break;\n        }\n\n        const bytes = change[2].length;\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const promise = this.#sendChange(change).finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n"],"mappings":";;;;;;;;;AAeA,IAAM,mCAAmC,KAAK,OAAO;AACrD,IAAM,kCAAkC;;;;;;;;AAcxC,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CAEA,YACE,iBACA,IACA,WACA,YACA,cACA,UAA6B,CAAC,GAC9B;EACA,KAAKA,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAKC,cAAc;EACnB,KAAKC,gBAAgB;EACrB,KAAKE,aAAa;EAClB,KAAKC,SAAS;EACd,KAAKC,WAAW,IAAI,WAAW;EAC/B,KAAKH,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAKC;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAKC;CACd;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAKD,YAAY;GAC/B,IAAI,KAAKE,UAAU;IAIjB,KAAKC,aAAa,MAAM;IACxB,OAAO,KAAKC,0BAA0B;GACxC;GACA,OAAO,KAAKC,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAKC,cAAc;GACtB,KAAKA,eAAe;GACpB,KAAK,WAAW,KAAKR,cAAc,CAAC;EACtC;CACF;CAEA,WAAW,QAAgB;EACzB,IAAI,KAAKF,oBAAoB,KAAK,KAAKU,cACrC,KAAUC,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAEhD;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAKC,YAAY;EACjB,MAAM,KAAKH,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAKG,YAAY;EACjB,IAAI,CAAC,KAAKN,UACR,OAAO,KAAKO,iBAAiB;EAE/B,IAAI,CAAC,KAAKA,eAAe;GAKvB,KAAKA,gBAAgB,KAAKC,cAAc;GACxC,KAAUD,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAKA;CACd;CAEA,MAAMJ,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAKL,aAAa;EAEpB,MAAM,SAAS,MAAM,KAAKW,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YACjC,KAAKV,SAAS,IAAI,KAAKA,QAAQ,SAAS;CAE5C;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAKU,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAMA,2BAA2B,MAAc;EAC7C,KAAKC;EACL,MAAM,EAAC,WAAU,KAAKf,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAKe;GACL,KAAKC;EACP;CACF;CAUA,WAAW;CACX,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAKD,WAAW,KAAKE;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAKD;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAKE,SAAS,UAAU,YAC7B,KAAKA,SAAS,MAAM;EAEtB,KAAKA,SAAS,KAAK;GAAC,WAAW,KAAKF;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAKE;EACA,MAAM,UAAU,KAAK;EACrB,IAAI,KAAKE,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAKD;GACd,cAAc,KAAKE;EACrB;EAEF,MAAM,OAAO,KAAKD,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAKA,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAKD;GACd,cAAc,KAAKE;EACrB;CACF;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAKpB,oBAAoB;EACpC;EACA,OAAO;CACT;CAEA,KAAK,KAAe;EAClB,KAAK,MAAM,GAAmB,OAAO,GAAG,CAAC;CAC3C;CAEA,MAAM,OAAmB,SAAkB;EAGzC,KAAKM,WAAW;EAChB,KAAKe,gBAAgB;EACrB,KAAKlB,qBAAqB,WAAW;EAErC,IAAI,OAEF,KAAUQ,gBAAgB,CAAC,SAAS;GAAC,MAAM;GAAO;EAAO,CAAC,CAAC,EAAE,cAC3D,KAAKV,YAAY,OAAO,CAC1B;OAEA,KAAKA,YAAY,OAAO;CAE5B;CAEA,IAAIiB,gBAAgB;EAClB,OAAO,KAAKZ,UAAU,QAAQ;CAChC;CAEA,IAAIc,wBAAwB;EAI1B,OAAO,KAAKC,gBAAgB,KAAKC;CACnC;CAEA,aAAa,QAA2B;EACtC,OAAO,KAAKhB,UAAU,gDAAgD;EACtE,KAAKA,SAAS,KAAK,MAAM;EACzB,KAAKe,iBAAiB,OAAO,GAAG;CAClC;CAEA,4BAA2C;EACzC,OAAO,KAAKlB,qBAAqB,aAAa,KAAKiB,qBAAqB;CAC1E;CAEA,MAAMN,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,SAAS,KAAKR,UAAU,MAAM;IACpC,IAAI,CAAC,QAAQ;KACX,KAAKA,WAAW;KAChB,KAAKe,gBAAgB;KACrB,KAAKlB,qBAAqB,uBACxB,KAAKiB,qBACP;KACA;IACF;IAEA,MAAM,QAAQ,OAAO,GAAG;IACxB,KAAKC,iBAAiB;IACtB,KAAKC,yBAAyB;IAC9B,KAAKnB,qBAAqB,uBACxB,KAAKiB,qBACP;IAKA,MAAM,UAAU,KAAKX,YAAY,MAAM,EAAE,cAAc;KACrD,KAAKa,yBAAyB;KAC9B,KAAKnB,qBAAqB,uBACxB,KAAKiB,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAKjB,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAKU,gBAAgB;GACrB,KAAKV,qBAAqB,uBACxB,KAAKiB,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAKG,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAKC,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAKA,SAAS,WAAW,KAAK,gBAAgB,KAAKD,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAKC,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF"}
+{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#latestStatus","#backlogBackpressure","#backlogFullWaiters","#onAck","#watermark","#acked","#backlog","#bufferedBacklogBytes","#pushBacklog","#maybeWaitForBacklogSpace","#sendChange","#initialized","#sendDownstream","#initialize","#backlogDrain","#drainBacklog","#sendStringifiedDownstream","#pending","#pendingBytes","#processed","#backlogCount","#samples","#totalBufferedBytes","#missedLastTimeout","#laggingSinceMs","#backlogBytes","#resolveBacklogFullWaiters","#backlogInFlightBytes","#lowWaterBytes","#waiters"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n};\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly #downstream: Subscription<string>;\n  readonly #latestStatus: () => Status;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<WatermarkedChange> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    watermark: string,\n    downstream: Subscription<string>,\n    latestStatus: () => Status,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.#downstream = downstream;\n    this.#latestStatus = latestStatus;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      this.sendStatus(this.#latestStatus());\n    }\n  }\n\n  sendStatus(status: Status) {\n    if (this.#initialized) {\n      void this.#sendDownstream(['status', status]);\n    }\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      // Sends can complete out of order (e.g. the bounded window in\n      // #drainBacklog), so the ack only advances monotonically, and listeners\n      // are only notified when it does.\n      const acked = max(this.#acked, watermark);\n      if (acked !== this.#acked) {\n        this.#acked = acked;\n        this.#onAck?.(acked);\n      }\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(error?: ErrorType, message?: string) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream(['error', {type: error, message}]).finally(() =>\n        this.#downstream.cancel(),\n      );\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(change: WatermarkedChange) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(change);\n    this.#backlogBytes += change[2].length;\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const change = this.#backlog?.shift();\n        if (!change) {\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          break;\n        }\n\n        const bytes = change[2].length;\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const promise = this.#sendChange(change).finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAmBA,IAAM,mCAAmC,KAAK,OAAO;AACrD,IAAM,kCAAkC;;;;;;;;AAmCxC,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CAEA,YACE,iBACA,IACA,WACA,YACA,cACA,UAA6B,CAAC,GAC9B;EACA,KAAKA,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAKC,cAAc;EACnB,KAAKC,gBAAgB;EACrB,KAAKI,aAAa;EAClB,KAAKC,SAAS;EACd,KAAKC,WAAW,IAAI,WAAW;EAC/B,KAAKL,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAKE,SAAS,QAAQ;CACxB;CAEA,IAAI,YAAY;EACd,OAAO,KAAKC;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAKC;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAKE,yBAAyB,KAAKN,qBAAqB;CAE5D;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAKC,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAKA,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAKE,YAAY;GAC/B,IAAI,KAAKE,UAAU;IAIjB,KAAKE,aAAa,MAAM;IACxB,OAAO,KAAKC,0BAA0B;GACxC;GACA,OAAO,KAAKC,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAKC,cAAc;GACtB,KAAKA,eAAe;GACpB,KAAK,WAAW,KAAKX,cAAc,CAAC;EACtC;CACF;CAEA,WAAW,QAAgB;EACzB,IAAI,KAAKW,cACP,KAAUC,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAEhD;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAKC,YAAY;EACjB,MAAM,KAAKH,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAKG,YAAY;EACjB,IAAI,CAAC,KAAKP,UACR,OAAO,KAAKQ,iBAAiB;EAE/B,IAAI,CAAC,KAAKA,eAAe;GAKvB,KAAKA,gBAAgB,KAAKC,cAAc;GACxC,KAAUD,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAKA;CACd;CAEA,MAAMJ,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAKN,aAAa;EAEpB,MAAM,SAAS,MAAM,KAAKY,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YAAY;GAI7C,MAAM,QAAQ,IAAI,KAAKX,QAAQ,SAAS;GACxC,IAAI,UAAU,KAAKA,QAAQ;IACzB,KAAKA,SAAS;IACd,KAAKF,SAAS,KAAK;GACrB;EACF;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAKa,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAMA,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAKC;EACL,KAAKC,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAKnB,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAKkB;GACL,KAAKC,iBAAiB;GACtB,KAAKC;EACP;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAKF,WAAW,KAAKG;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAKD;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAKE,SAAS,UAAU,YAC7B,KAAKA,SAAS,MAAM;EAEtB,KAAKA,SAAS,KAAK;GAAC,WAAW,KAAKF;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAKE,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAKD;GACd,cAAc,KAAKb;GACnB,oBAAoB,KAAKe;GACzB,mBAAmB,KAAKC;EAC1B;EAEF,MAAM,OAAO,KAAKF,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAKA,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAKD;GACd,cAAc,KAAKb;GACnB,oBAAoB,KAAKe;GACzB,mBAAmB,KAAKC;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAKA,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAKC,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAKD,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAKC,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAKA,oBAAoB;EACzB,OAAO,MAAM,KAAKA;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAK1B,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MAAM,OAAmB,SAAkB;EAGzC,KAAKQ,WAAW;EAChB,KAAKmB,gBAAgB;EACrB,KAAKxB,qBAAqB,WAAW;EAGrC,KAAKyB,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAUd,gBAAgB,CAAC,SAAS;GAAC,MAAM;GAAO;EAAO,CAAC,CAAC,EAAE,cAC3D,KAAKb,YAAY,OAAO,CAC1B;OAEA,KAAKA,YAAY,OAAO;CAE5B;CAEA,IAAIqB,gBAAgB;EAClB,OAAO,KAAKd,UAAU,QAAQ;CAChC;CAEA,IAAIC,wBAAwB;EAI1B,OAAO,KAAKkB,gBAAgB,KAAKE;CACnC;CAEA,IAAIL,sBAAsB;EACxB,OAAO,KAAKf,wBAAwB,KAAKW;CAC3C;CAEA,aAAa,QAA2B;EACtC,OAAO,KAAKZ,UAAU,gDAAgD;EACtE,KAAKA,SAAS,KAAK,MAAM;EACzB,KAAKmB,iBAAiB,OAAO,GAAG;EAChC,IAAI,KAAK,aACP,KAAKC,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAKxB,mBAAmB;EAC5C,KAAKA,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAKD,qBAAqB,aAAa,KAAKM,qBAAqB;CAC1E;CAEA,MAAMQ,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,SAAS,KAAKT,UAAU,MAAM;IACpC,IAAI,CAAC,QAAQ;KACX,KAAKA,WAAW;KAChB,KAAKmB,gBAAgB;KACrB,KAAKxB,qBAAqB,uBACxB,KAAKM,qBACP;KACA;IACF;IAEA,MAAM,QAAQ,OAAO,GAAG;IACxB,KAAKkB,iBAAiB;IACtB,KAAKE,yBAAyB;IAC9B,KAAK1B,qBAAqB,uBACxB,KAAKM,qBACP;IAKA,MAAM,UAAU,KAAKG,YAAY,MAAM,EAAE,cAAc;KACrD,KAAKiB,yBAAyB;KAC9B,KAAK1B,qBAAqB,uBACxB,KAAKM,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAKN,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAKa,gBAAgB;GACrB,KAAKb,qBAAqB,uBACxB,KAAKM,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAKqB,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAKC,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAKA,SAAS,WAAW,KAAK,gBAAgB,KAAKD,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAKC,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF"}

package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts

@@ -0,0 +1,34 @@
+import type { Sink } from '../../types/streams.ts';
+import type { ChangeSourceUpstream, ChangeStreamMessage } from '../change-source/protocol/current.ts';
+type Opts = {
+    trackPgChangeLog: boolean;
+    trackBackup: boolean;
+};
+/**
+ * Tracks the progress (watermark) of multiple streams:
+ * - downstream transactions and status messages (the latter
+ *   indicating LSNs that are not relevant to the publication
+ *   but nevertheless need to be ACKed)
+ * - PG change-log commits
+ * - backup watermarks
+ *
+ * and sends upstream ACKs accordingly. When both the PG change-log
+ * and backup watermarks are being considered for upstream ACKs
+ * (i.e. RMv1.5), only watermarks that have been reached by both
+ * stores are acked.
+ *
+ * The UpstreamAcker also takes into account that an upstream
+ * connection can be disconnected and {@link reset()}. In this case,
+ * the progress of the persistent stores is retained and the acks
+ * are resent on the new upstream connection as necessary.
+ */
+export declare class UpstreamAcker {
+    #private;
+    constructor({ trackPgChangeLog, trackBackup }: Opts);
+    reset(upstream: Sink<ChangeSourceUpstream>): void;
+    trackDownstream(downstream: ChangeStreamMessage): void;
+    trackPgChangeLog(committedWatermark: string): void;
+    trackBackup(backedUpWatermark: string): void;
+}
+export {};
+//# sourceMappingURL=upstream-acker.d.ts.map

package/out/zero-cache/src/services/change-streamer/upstream-acker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"upstream-acker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/upstream-acker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,wBAAwB,CAAC;AACjD,OAAO,KAAK,EACV,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,sCAAsC,CAAC;AAE9C,KAAK,IAAI,GAAG;IACV,gBAAgB,EAAE,OAAO,CAAC;IAC1B,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,aAAa;;gBAYZ,EAAC,gBAAgB,EAAE,WAAW,EAAC,EAAE,IAAI;IASjD,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,oBAAoB,CAAC;IAO1C,eAAe,CAAC,UAAU,EAAE,mBAAmB;IAgB/C,gBAAgB,CAAC,kBAAkB,EAAE,MAAM;IAS3C,WAAW,CAAC,iBAAiB,EAAE,MAAM;CA+BtC"}

package/out/zero-cache/src/services/change-streamer/upstream-acker.js

@@ -0,0 +1,86 @@
+import { assert } from "../../../../shared/src/asserts.js";
+import { max, min } from "../../types/lexi-version.js";
+//#region ../zero-cache/src/services/change-streamer/upstream-acker.ts
+/**
+* Tracks the progress (watermark) of multiple streams:
+* - downstream transactions and status messages (the latter
+*   indicating LSNs that are not relevant to the publication
+*   but nevertheless need to be ACKed)
+* - PG change-log commits
+* - backup watermarks
+*
+* and sends upstream ACKs accordingly. When both the PG change-log
+* and backup watermarks are being considered for upstream ACKs
+* (i.e. RMv1.5), only watermarks that have been reached by both
+* stores are acked.
+*
+* The UpstreamAcker also takes into account that an upstream
+* connection can be disconnected and {@link reset()}. In this case,
+* the progress of the persistent stores is retained and the acks
+* are resent on the new upstream connection as necessary.
+*/
+var UpstreamAcker = class {
+	#trackPgChangeLog;
+	#trackBackup;
+	#pgChangeLogWatermark = "";
+	#backupWatermark = "";
+	#upstream;
+	#lastTx = "";
+	#lastStatus = "";
+	#lastAck = "";
+	constructor({ trackPgChangeLog, trackBackup }) {
+		assert(trackPgChangeLog || trackBackup, `At least one of trackPgChangeLog or trackBackup must be true`);
+		this.#trackPgChangeLog = trackPgChangeLog;
+		this.#trackBackup = trackBackup;
+	}
+	reset(upstream) {
+		this.#upstream = upstream;
+		this.#lastTx = "";
+		this.#lastStatus = "";
+		this.#lastAck = "";
+	}
+	trackDownstream(downstream) {
+		const [tag, msg] = downstream;
+		switch (tag) {
+			case "status":
+				if (msg.ack) this.#lastStatus = downstream[2].watermark;
+				this.#maybeAck();
+				break;
+			case "commit":
+				this.#lastTx = downstream[2].watermark;
+				this.#maybeAck();
+				break;
+		}
+	}
+	trackPgChangeLog(committedWatermark) {
+		this.#pgChangeLogWatermark = max(committedWatermark, this.#pgChangeLogWatermark);
+		this.#maybeAck();
+	}
+	trackBackup(backedUpWatermark) {
+		this.#backupWatermark = max(backedUpWatermark, this.#backupWatermark);
+		this.#maybeAck();
+	}
+	#maybeAck() {
+		const currentWatermark = !this.#trackBackup ? this.#pgChangeLogWatermark : !this.#trackPgChangeLog ? this.#backupWatermark : min(this.#pgChangeLogWatermark, this.#backupWatermark);
+		if (currentWatermark > this.#lastAck) {
+			this.#upstream?.push([
+				"status",
+				{ tag: "commit" },
+				{ watermark: currentWatermark }
+			]);
+			this.#lastAck = currentWatermark;
+		}
+		if (this.#lastAck >= this.#lastTx && this.#lastStatus > this.#lastAck) {
+			this.#upstream?.push([
+				"status",
+				{ ack: true },
+				{ watermark: this.#lastStatus }
+			]);
+			this.#lastAck = this.#lastStatus;
+		}
+	}
+};
+//#endregion
+export { UpstreamAcker };
+
+//# sourceMappingURL=upstream-acker.js.map

package/out/zero-cache/src/services/change-streamer/upstream-acker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"upstream-acker.js","names":["#trackPgChangeLog","#trackBackup","#upstream","#lastTx","#lastStatus","#lastAck","#maybeAck","#pgChangeLogWatermark","#backupWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/upstream-acker.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport {max, min} from '../../types/lexi-version.ts';\nimport type {Sink} from '../../types/streams.ts';\nimport type {\n  ChangeSourceUpstream,\n  ChangeStreamMessage,\n} from '../change-source/protocol/current.ts';\n\ntype Opts = {\n  trackPgChangeLog: boolean;\n  trackBackup: boolean;\n};\n\n/**\n * Tracks the progress (watermark) of multiple streams:\n * - downstream transactions and status messages (the latter\n *   indicating LSNs that are not relevant to the publication\n *   but nevertheless need to be ACKed)\n * - PG change-log commits\n * - backup watermarks\n *\n * and sends upstream ACKs accordingly. When both the PG change-log\n * and backup watermarks are being considered for upstream ACKs\n * (i.e. RMv1.5), only watermarks that have been reached by both\n * stores are acked.\n *\n * The UpstreamAcker also takes into account that an upstream\n * connection can be disconnected and {@link reset()}. In this case,\n * the progress of the persistent stores is retained and the acks\n * are resent on the new upstream connection as necessary.\n */\nexport class UpstreamAcker {\n  readonly #trackPgChangeLog: boolean;\n  readonly #trackBackup: boolean;\n\n  #pgChangeLogWatermark = '';\n  #backupWatermark = '';\n\n  #upstream: Sink<ChangeSourceUpstream> | undefined;\n  #lastTx = '';\n  #lastStatus = '';\n  #lastAck = '';\n\n  constructor({trackPgChangeLog, trackBackup}: Opts) {\n    assert(\n      trackPgChangeLog || trackBackup,\n      `At least one of trackPgChangeLog or trackBackup must be true`,\n    );\n    this.#trackPgChangeLog = trackPgChangeLog;\n    this.#trackBackup = trackBackup;\n  }\n\n  reset(upstream: Sink<ChangeSourceUpstream>) {\n    this.#upstream = upstream;\n    this.#lastTx = '';\n    this.#lastStatus = '';\n    this.#lastAck = '';\n  }\n\n  trackDownstream(downstream: ChangeStreamMessage) {\n    const [tag, msg] = downstream;\n    switch (tag) {\n      case 'status':\n        if (msg.ack) {\n          this.#lastStatus = downstream[2].watermark;\n        }\n        this.#maybeAck();\n        break;\n      case 'commit':\n        this.#lastTx = downstream[2].watermark;\n        this.#maybeAck();\n        break;\n    }\n  }\n\n  trackPgChangeLog(committedWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#pgChangeLogWatermark = max(\n      committedWatermark,\n      this.#pgChangeLogWatermark,\n    );\n    this.#maybeAck();\n  }\n\n  trackBackup(backedUpWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#backupWatermark = max(backedUpWatermark, this.#backupWatermark);\n    this.#maybeAck();\n  }\n\n  #maybeAck() {\n    const currentWatermark = !this.#trackBackup\n      ? this.#pgChangeLogWatermark\n      : !this.#trackPgChangeLog\n        ? this.#backupWatermark\n        : min(this.#pgChangeLogWatermark, this.#backupWatermark);\n    if (currentWatermark > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {tag: 'commit'},\n        {watermark: currentWatermark},\n      ]);\n      this.#lastAck = currentWatermark;\n    }\n    // If all committed transactions have been acked, ack any outstanding\n    // status LSN's thereafter (i.e. non-publication upstream changes).\n    if (this.#lastAck >= this.#lastTx && this.#lastStatus > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {ack: true},\n        {watermark: this.#lastStatus},\n      ]);\n      this.#lastAck = this.#lastStatus;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+BA,IAAa,gBAAb,MAA2B;CACzB;CACA;CAEA,wBAAwB;CACxB,mBAAmB;CAEnB;CACA,UAAU;CACV,cAAc;CACd,WAAW;CAEX,YAAY,EAAC,kBAAkB,eAAoB;EACjD,OACE,oBAAoB,aACpB,8DACF;EACA,KAAKA,oBAAoB;EACzB,KAAKC,eAAe;CACtB;CAEA,MAAM,UAAsC;EAC1C,KAAKC,YAAY;EACjB,KAAKC,UAAU;EACf,KAAKC,cAAc;EACnB,KAAKC,WAAW;CAClB;CAEA,gBAAgB,YAAiC;EAC/C,MAAM,CAAC,KAAK,OAAO;EACnB,QAAQ,KAAR;GACE,KAAK;IACH,IAAI,IAAI,KACN,KAAKD,cAAc,WAAW,GAAG;IAEnC,KAAKE,UAAU;IACf;GACF,KAAK;IACH,KAAKH,UAAU,WAAW,GAAG;IAC7B,KAAKG,UAAU;IACf;EACJ;CACF;CAEA,iBAAiB,oBAA4B;EAE3C,KAAKC,wBAAwB,IAC3B,oBACA,KAAKA,qBACP;EACA,KAAKD,UAAU;CACjB;CAEA,YAAY,mBAA2B;EAErC,KAAKE,mBAAmB,IAAI,mBAAmB,KAAKA,gBAAgB;EACpE,KAAKF,UAAU;CACjB;CAEA,YAAY;EACV,MAAM,mBAAmB,CAAC,KAAKL,eAC3B,KAAKM,wBACL,CAAC,KAAKP,oBACJ,KAAKQ,mBACL,IAAI,KAAKD,uBAAuB,KAAKC,gBAAgB;EAC3D,IAAI,mBAAmB,KAAKH,UAAU;GACpC,KAAKH,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,SAAQ;IACd,EAAC,WAAW,iBAAgB;GAC9B,CAAC;GACD,KAAKG,WAAW;EAClB;EAGA,IAAI,KAAKA,YAAY,KAAKF,WAAW,KAAKC,cAAc,KAAKC,UAAU;GACrE,KAAKH,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,KAAI;IACV,EAAC,WAAW,KAAKE,YAAW;GAC9B,CAAC;GACD,KAAKC,WAAW,KAAKD;EACvB;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts

@@ -0,0 +1,29 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Source } from '../../types/streams.ts';
+import type { BackedUpWatermark } from './backup-monitor.ts';
+export type VfsPollerConfig = {
+    executable: string;
+    remotePollIntervalMs: number;
+    remoteQueryTimeoutMs?: number;
+    backupURL: string;
+    region?: string | undefined;
+    endpoint?: string | undefined;
+    logLevel?: string;
+    logFormat?: string;
+};
+export declare class VfsWatermarkPoller {
+    #private;
+    constructor(lc: LogContext, replicaFile: string, pollerConfig: VfsPollerConfig);
+    /**
+     * Starts polling for watermarks and pushes new ones to returned stream.
+     * Must only be called once.
+     */
+    start(): Source<BackedUpWatermark>;
+    readonly checkLocalWatermark: () => void;
+}
+export declare function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region, }: {
+    readonly backupURL: string;
+    readonly endpoint?: string | undefined;
+    readonly region?: string | undefined;
+}): string;
+//# sourceMappingURL=vfs-watermark-poller.d.ts.map

package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-poller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/vfs-watermark-poller.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAe3D,MAAM,MAAM,eAAe,GAAG;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAyBF,qBAAa,kBAAkB;;gBAmB3B,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,eAAe;IAyB/B;;;OAGG;IACH,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC;IAYlC,QAAQ,CAAC,mBAAmB,aAiB1B;CA+GH;AAED,wBAAgB,4BAA4B,CAAC,EAC3C,SAAS,EACT,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,GAAG,MAAM,CAiBT"}

package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js

@@ -0,0 +1,154 @@
+import { assert } from "../../../../shared/src/asserts.js";
+import { parse, valita_exports } from "../../../../shared/src/valita.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { RunningState } from "../running-state.js";
+import { Subscription } from "../../types/subscription.js";
+import { spawn } from "node:child_process";
+import { createInterface } from "node:readline";
+//#region ../zero-cache/src/services/change-streamer/vfs-watermark-poller.ts
+var LOCAL_CHECK_INTERVAL_MS = 1e3;
+var queryResultRowSchema = valita_exports.object({
+	stateVersion: valita_exports.string(),
+	writeTimeMs: valita_exports.number()
+});
+var remoteQueryMetadataSchema = valita_exports.object({ litestream_time: valita_exports.string().map((str) => new Date(str).getTime()).optional() });
+var remoteQueryLineSchema = valita_exports.object({
+	query_result: valita_exports.tuple([queryResultRowSchema]),
+	metadata: remoteQueryMetadataSchema
+});
+var INITIAL_BACKOFF_MS = 2e3;
+var MAX_BACKOFF_MS = 3e4;
+var VfsWatermarkPoller = class {
+	#lc;
+	#state = new RunningState("vfs-watermark-poller");
+	#pollerConfig;
+	#readLocalWatermark;
+	#stream;
+	#localWatermark;
+	#remoteWatermark;
+	#localPollTimer;
+	#shouldPollRemote = false;
+	#remotePoller;
+	#remotePollerBackoffMs = INITIAL_BACKOFF_MS;
+	#remotePollerBackoffTimer;
+	constructor(lc, replicaFile, pollerConfig) {
+		this.#lc = lc.withContext("component", "vfs-watermark-poller");
+		this.#pollerConfig = pollerConfig;
+		const db = new Database(this.#lc, replicaFile, { readonly: true });
+		this.#readLocalWatermark = db.prepare(`SELECT stateVersion, writeTimeMs FROM "_zero.replicationState"`);
+		this.#localWatermark = parse(this.#readLocalWatermark.get(), queryResultRowSchema);
+		this.#stream = Subscription.create({ cleanup: () => {
+			this.#disableRemotePoller();
+			clearInterval(this.#localPollTimer);
+			clearTimeout(this.#remotePollerBackoffTimer);
+			this.#state.stop(this.#lc);
+			db.close();
+		} });
+	}
+	/**
+	* Starts polling for watermarks and pushes new ones to returned stream.
+	* Must only be called once.
+	*/
+	start() {
+		assert(this.#localPollTimer === void 0, `Already called start()`);
+		this.#lc.info?.(`monitoring backups at ${this.#pollerConfig.backupURL}`);
+		this.#localPollTimer = setInterval(this.checkLocalWatermark, LOCAL_CHECK_INTERVAL_MS);
+		this.checkLocalWatermark();
+		return this.#stream;
+	}
+	checkLocalWatermark = () => {
+		try {
+			this.#localWatermark = parse(this.#readLocalWatermark.get(), queryResultRowSchema);
+		} catch (e) {
+			this.#lc.warn?.(`unable to read local watermark`, e);
+			return;
+		}
+		if (this.#localWatermark.stateVersion !== this.#remoteWatermark?.watermark) this.#enableRemotePoller();
+		else this.#disableRemotePoller();
+	};
+	#enableRemotePoller() {
+		if (this.#remotePoller || this.#remotePollerBackoffTimer) return;
+		this.#shouldPollRemote = true;
+		if (this.#remoteWatermark) this.#lc.debug?.(`local and remote watermarks differ. resuming backup polling`, {
+			local: this.#localWatermark,
+			remote: this.#remoteWatermark
+		});
+		const child = spawn(this.#pollerConfig.executable, [
+			"--replica-url",
+			buildLitestreamVfsReplicaURL(this.#pollerConfig),
+			"--query",
+			`SELECT stateVersion, writeTimeMs FROM "_zero.replicationState"`,
+			"--remote-poll-interval",
+			`${this.#pollerConfig.remotePollIntervalMs}ms`,
+			...this.#pollerConfig.logLevel ? ["--log-level", this.#pollerConfig.logLevel] : [],
+			...this.#pollerConfig.logFormat ? ["--log-format", this.#pollerConfig.logFormat] : [],
+			...this.#pollerConfig.remoteQueryTimeoutMs ? ["--query-timeout", `${this.#pollerConfig.remoteQueryTimeoutMs}ms`] : []
+		], { stdio: [
+			"pipe",
+			"pipe",
+			"inherit"
+		] }).on("error", (err) => {
+			this.#lc.error?.(`received error from vfs-query process`, err);
+		}).on("close", (code, signal) => {
+			this.#remotePoller = void 0;
+			if (this.#shouldPollRemote) {
+				if (code !== 0) this.#lc.warn?.(`vfs-query exited with ${code}, ${signal}. retrying in ${this.#remotePollerBackoffMs}ms`);
+				if (!this.#remotePollerBackoffTimer) {
+					this.#remotePollerBackoffTimer = setTimeout(() => {
+						this.#remotePollerBackoffTimer = void 0;
+						if (this.#shouldPollRemote) this.#enableRemotePoller();
+					}, this.#remotePollerBackoffMs);
+					this.#remotePollerBackoffMs = Math.min(this.#remotePollerBackoffMs * 2, MAX_BACKOFF_MS);
+				}
+			}
+		});
+		createInterface({ input: child.stdout }).on("line", this.#readRemotePollerLine);
+		this.#remotePoller = child;
+	}
+	#readRemotePollerLine = (line) => {
+		try {
+			const update = parse(JSON.parse(line), remoteQueryLineSchema, "passthrough");
+			const [{ stateVersion: watermark, writeTimeMs }] = update.query_result;
+			const remoteWatermark = {
+				watermark,
+				writeTimeMs,
+				backupTimeMs: update.metadata.litestream_time ?? Date.now()
+			};
+			if (remoteWatermark.watermark !== this.#remoteWatermark?.watermark) {
+				this.#lc.debug?.(`backup watermark advanced from ${this.#remoteWatermark?.watermark} to ${remoteWatermark.watermark}`, {
+					local: this.#localWatermark,
+					remote: remoteWatermark
+				});
+				this.#stream.push(remoteWatermark);
+			}
+			this.#remoteWatermark = remoteWatermark;
+			this.#remotePollerBackoffMs = INITIAL_BACKOFF_MS;
+		} catch (e) {
+			this.#lc.error?.(`unable to parse vfs-query output: ${line}`, e);
+		}
+	};
+	#disableRemotePoller() {
+		if (!this.#shouldPollRemote) return;
+		this.#shouldPollRemote = false;
+		if (this.#remotePoller || this.#remotePollerBackoffTimer) this.#lc.debug?.(`local and remote watermarks match. pausing backup polling`, {
+			local: this.#localWatermark,
+			remote: this.#remoteWatermark
+		});
+		this.#remotePoller?.kill("SIGTERM");
+		this.#remotePoller = void 0;
+		clearTimeout(this.#remotePollerBackoffTimer);
+		this.#remotePollerBackoffTimer = void 0;
+	}
+};
+function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region }) {
+	if (endpoint === void 0 && region === void 0) return backupURL;
+	const url = new URL(backupURL);
+	if (url.protocol !== "s3:") return backupURL;
+	if (endpoint !== void 0 && !url.searchParams.has("endpoint")) url.searchParams.set("endpoint", endpoint);
+	if (region !== void 0 && !url.searchParams.has("region")) url.searchParams.set("region", region);
+	return url.toString();
+}
+//#endregion
+export { VfsWatermarkPoller };
+
+//# sourceMappingURL=vfs-watermark-poller.js.map

package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-poller.js","names":["#lc","#state","#pollerConfig","#readLocalWatermark","#stream","#localWatermark","#disableRemotePoller","#localPollTimer","#remotePollerBackoffTimer","#remoteWatermark","#enableRemotePoller","#remotePoller","#shouldPollRemote","#remotePollerBackoffMs","#readRemotePollerLine"],"sources":["../../../../../../zero-cache/src/services/change-streamer/vfs-watermark-poller.ts"],"sourcesContent":["import {spawn, type ChildProcess} from 'node:child_process';\nimport {createInterface} from 'node:readline';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database, type Statement} from '../../../../zqlite/src/db.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nconst LOCAL_CHECK_INTERVAL_MS = 1_000;\n\ntype LocalWatermark = {\n  stateVersion: string;\n  writeTimeMs: number;\n};\n\ntype RemoteWatermark = {\n  watermark: string;\n  writeTimeMs: number;\n  backupTimeMs: number;\n};\n\nexport type VfsPollerConfig = {\n  executable: string;\n  remotePollIntervalMs: number;\n  remoteQueryTimeoutMs?: number;\n  backupURL: string;\n  region?: string | undefined;\n  endpoint?: string | undefined;\n  logLevel?: string;\n  logFormat?: string;\n};\n\nconst queryResultRowSchema = v.object({\n  stateVersion: v.string(),\n  writeTimeMs: v.number(),\n});\n\nconst remoteQueryMetadataSchema = v.object({\n  litestream_time: v\n    .string()\n    .map(str => new Date(str).getTime())\n    .optional(),\n});\n\n// Expected format of each line of stdout from the vfs-query, e.g.\n// {\"query_result\":[{\"stateVersion\":\"51y50ko\",\"writeTimeMs\":1787259951121}],\"metadata\":{\"litestream_time\":\"2026-08-20T21:05:57.36439Z\"}}\n// {\"query_result\":[{\"stateVersion\":\"51y7360\",\"writeTimeMs\":1787259966137}],\"metadata\":{\"litestream_time\":\"2026-08-20T21:06:12.384531Z\"}}\nconst remoteQueryLineSchema = v.object({\n  query_result: v.tuple([queryResultRowSchema]),\n  metadata: remoteQueryMetadataSchema,\n});\n\nconst INITIAL_BACKOFF_MS = 2_000;\nconst MAX_BACKOFF_MS = 30_000;\n\nexport class VfsWatermarkPoller {\n  readonly #lc: LogContext;\n  readonly #state = new RunningState('vfs-watermark-poller');\n  readonly #pollerConfig: VfsPollerConfig;\n  readonly #readLocalWatermark: Statement;\n\n  readonly #stream: Subscription<BackedUpWatermark>;\n\n  #localWatermark: LocalWatermark;\n  #remoteWatermark: RemoteWatermark | undefined;\n  #localPollTimer: NodeJS.Timeout | undefined;\n\n  #shouldPollRemote = false;\n\n  #remotePoller: ChildProcess | undefined;\n  #remotePollerBackoffMs = INITIAL_BACKOFF_MS;\n  #remotePollerBackoffTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    pollerConfig: VfsPollerConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'vfs-watermark-poller');\n    this.#pollerConfig = pollerConfig;\n\n    const db = new Database(this.#lc, replicaFile, {readonly: true});\n    this.#readLocalWatermark = db.prepare(\n      `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n    );\n    // Purely to type the member variable as always defined\n    this.#localWatermark = v.parse(\n      this.#readLocalWatermark.get(),\n      queryResultRowSchema,\n    );\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        this.#disableRemotePoller();\n        clearInterval(this.#localPollTimer);\n        clearTimeout(this.#remotePollerBackoffTimer);\n        this.#state.stop(this.#lc);\n        db.close();\n      },\n    });\n  }\n\n  /**\n   * Starts polling for watermarks and pushes new ones to returned stream.\n   * Must only be called once.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#localPollTimer === undefined, `Already called start()`);\n    this.#lc.info?.(`monitoring backups at ${this.#pollerConfig.backupURL}`);\n    this.#localPollTimer = setInterval(\n      this.checkLocalWatermark,\n      LOCAL_CHECK_INTERVAL_MS,\n    );\n    this.checkLocalWatermark();\n    return this.#stream;\n  }\n\n  // Exported for testing\n  readonly checkLocalWatermark = () => {\n    try {\n      this.#localWatermark = v.parse(\n        this.#readLocalWatermark.get(),\n        queryResultRowSchema,\n      );\n    } catch (e) {\n      this.#lc.warn?.(`unable to read local watermark`, e);\n      return;\n    }\n    if (\n      this.#localWatermark.stateVersion !== this.#remoteWatermark?.watermark\n    ) {\n      this.#enableRemotePoller();\n    } else {\n      this.#disableRemotePoller();\n    }\n  };\n\n  #enableRemotePoller() {\n    if (this.#remotePoller || this.#remotePollerBackoffTimer) {\n      return; // already polling or backing off\n    }\n    this.#shouldPollRemote = true;\n    if (this.#remoteWatermark) {\n      this.#lc.debug?.(\n        `local and remote watermarks differ. resuming backup polling`,\n        {local: this.#localWatermark, remote: this.#remoteWatermark},\n      );\n    }\n    const child = spawn(\n      this.#pollerConfig.executable,\n      [\n        '--replica-url',\n        buildLitestreamVfsReplicaURL(this.#pollerConfig),\n        '--query',\n        `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n        '--remote-poll-interval',\n        `${this.#pollerConfig.remotePollIntervalMs}ms`,\n        ...(this.#pollerConfig.logLevel\n          ? ['--log-level', this.#pollerConfig.logLevel]\n          : []),\n        ...(this.#pollerConfig.logFormat\n          ? ['--log-format', this.#pollerConfig.logFormat]\n          : []),\n        ...(this.#pollerConfig.remoteQueryTimeoutMs\n          ? ['--query-timeout', `${this.#pollerConfig.remoteQueryTimeoutMs}ms`]\n          : []),\n      ],\n      {stdio: ['pipe', 'pipe', 'inherit']},\n    )\n      .on('error', err => {\n        this.#lc.error?.(`received error from vfs-query process`, err);\n      })\n      .on('close', (code, signal) => {\n        this.#remotePoller = undefined;\n        if (this.#shouldPollRemote) {\n          if (code !== 0) {\n            this.#lc.warn?.(\n              `vfs-query exited with ${code}, ${signal}. retrying in ${this.#remotePollerBackoffMs}ms`,\n            );\n          }\n          if (!this.#remotePollerBackoffTimer) {\n            this.#remotePollerBackoffTimer = setTimeout(() => {\n              this.#remotePollerBackoffTimer = undefined;\n              if (this.#shouldPollRemote) {\n                this.#enableRemotePoller();\n              }\n            }, this.#remotePollerBackoffMs);\n            this.#remotePollerBackoffMs = Math.min(\n              this.#remotePollerBackoffMs * 2,\n              MAX_BACKOFF_MS,\n            );\n          }\n        }\n      });\n    createInterface({input: child.stdout}).on(\n      'line',\n      this.#readRemotePollerLine,\n    );\n    this.#remotePoller = child;\n  }\n\n  readonly #readRemotePollerLine = (line: string) => {\n    try {\n      const update = v.parse(\n        JSON.parse(line),\n        remoteQueryLineSchema,\n        'passthrough',\n      );\n      const [{stateVersion: watermark, writeTimeMs}] = update.query_result;\n      const remoteWatermark: RemoteWatermark = {\n        watermark,\n        writeTimeMs,\n        backupTimeMs: update.metadata.litestream_time ?? Date.now(),\n      };\n      if (remoteWatermark.watermark !== this.#remoteWatermark?.watermark) {\n        this.#lc.debug?.(\n          `backup watermark advanced from ${this.#remoteWatermark?.watermark} to ${remoteWatermark.watermark}`,\n          {local: this.#localWatermark, remote: remoteWatermark},\n        );\n        this.#stream.push(remoteWatermark);\n      }\n      this.#remoteWatermark = remoteWatermark;\n\n      // Reset backoff on successful output from the poller\n      this.#remotePollerBackoffMs = INITIAL_BACKOFF_MS;\n    } catch (e) {\n      this.#lc.error?.(`unable to parse vfs-query output: ${line}`, e);\n    }\n  };\n\n  #disableRemotePoller() {\n    if (!this.#shouldPollRemote) {\n      return;\n    }\n    this.#shouldPollRemote = false;\n    if (this.#remotePoller || this.#remotePollerBackoffTimer) {\n      this.#lc.debug?.(\n        `local and remote watermarks match. pausing backup polling`,\n        {local: this.#localWatermark, remote: this.#remoteWatermark},\n      );\n    }\n    this.#remotePoller?.kill('SIGTERM');\n    this.#remotePoller = undefined;\n    clearTimeout(this.#remotePollerBackoffTimer);\n    this.#remotePollerBackoffTimer = undefined;\n  }\n}\n\nexport function buildLitestreamVfsReplicaURL({\n  backupURL,\n  endpoint,\n  region,\n}: {\n  readonly backupURL: string;\n  readonly endpoint?: string | undefined;\n  readonly region?: string | undefined;\n}): string {\n  if (endpoint === undefined && region === undefined) {\n    return backupURL;\n  }\n\n  const url = new URL(backupURL);\n  if (url.protocol !== 's3:') {\n    return backupURL;\n  }\n\n  if (endpoint !== undefined && !url.searchParams.has('endpoint')) {\n    url.searchParams.set('endpoint', endpoint);\n  }\n  if (region !== undefined && !url.searchParams.has('region')) {\n    url.searchParams.set('region', region);\n  }\n  return url.toString();\n}\n"],"mappings":";;;;;;;;AAWA,IAAM,0BAA0B;AAwBhC,IAAM,uBAAuB,eAAE,OAAO;CACpC,cAAc,eAAE,OAAO;CACvB,aAAa,eAAE,OAAO;AACxB,CAAC;AAED,IAAM,4BAA4B,eAAE,OAAO,EACzC,iBAAiB,eACd,OAAO,EACP,KAAI,QAAO,IAAI,KAAK,GAAG,EAAE,QAAQ,CAAC,EAClC,SAAS,EACd,CAAC;AAKD,IAAM,wBAAwB,eAAE,OAAO;CACrC,cAAc,eAAE,MAAM,CAAC,oBAAoB,CAAC;CAC5C,UAAU;AACZ,CAAC;AAED,IAAM,qBAAqB;AAC3B,IAAM,iBAAiB;AAEvB,IAAa,qBAAb,MAAgC;CAC9B;CACA,SAAkB,IAAI,aAAa,sBAAsB;CACzD;CACA;CAEA;CAEA;CACA;CACA;CAEA,oBAAoB;CAEpB;CACA,yBAAyB;CACzB;CAEA,YACE,IACA,aACA,cACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAKE,gBAAgB;EAErB,MAAM,KAAK,IAAI,SAAS,KAAKF,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,KAAKG,sBAAsB,GAAG,QAC5B,gEACF;EAEA,KAAKE,kBAAkB,MACrB,KAAKF,oBAAoB,IAAI,GAC7B,oBACF;EACA,KAAKC,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,KAAKE,qBAAqB;GAC1B,cAAc,KAAKC,eAAe;GAClC,aAAa,KAAKC,yBAAyB;GAC3C,KAAKP,OAAO,KAAK,KAAKD,GAAG;GACzB,GAAG,MAAM;EACX,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKO,oBAAoB,KAAA,GAAW,wBAAwB;EACnE,KAAKP,IAAI,OAAO,yBAAyB,KAAKE,cAAc,WAAW;EACvE,KAAKK,kBAAkB,YACrB,KAAK,qBACL,uBACF;EACA,KAAK,oBAAoB;EACzB,OAAO,KAAKH;CACd;CAGA,4BAAqC;EACnC,IAAI;GACF,KAAKC,kBAAkB,MACrB,KAAKF,oBAAoB,IAAI,GAC7B,oBACF;EACF,SAAS,GAAG;GACV,KAAKH,IAAI,OAAO,kCAAkC,CAAC;GACnD;EACF;EACA,IACE,KAAKK,gBAAgB,iBAAiB,KAAKI,kBAAkB,WAE7D,KAAKC,oBAAoB;OAEzB,KAAKJ,qBAAqB;CAE9B;CAEA,sBAAsB;EACpB,IAAI,KAAKK,iBAAiB,KAAKH,2BAC7B;EAEF,KAAKI,oBAAoB;EACzB,IAAI,KAAKH,kBACP,KAAKT,IAAI,QACP,+DACA;GAAC,OAAO,KAAKK;GAAiB,QAAQ,KAAKI;EAAgB,CAC7D;EAEF,MAAM,QAAQ,MACZ,KAAKP,cAAc,YACnB;GACE;GACA,6BAA6B,KAAKA,aAAa;GAC/C;GACA;GACA;GACA,GAAG,KAAKA,cAAc,qBAAqB;GAC3C,GAAI,KAAKA,cAAc,WACnB,CAAC,eAAe,KAAKA,cAAc,QAAQ,IAC3C,CAAC;GACL,GAAI,KAAKA,cAAc,YACnB,CAAC,gBAAgB,KAAKA,cAAc,SAAS,IAC7C,CAAC;GACL,GAAI,KAAKA,cAAc,uBACnB,CAAC,mBAAmB,GAAG,KAAKA,cAAc,qBAAqB,GAAG,IAClE,CAAC;EACP,GACA,EAAC,OAAO;GAAC;GAAQ;GAAQ;EAAS,EAAC,CACrC,EACG,GAAG,UAAS,QAAO;GAClB,KAAKF,IAAI,QAAQ,yCAAyC,GAAG;EAC/D,CAAC,EACA,GAAG,UAAU,MAAM,WAAW;GAC7B,KAAKW,gBAAgB,KAAA;GACrB,IAAI,KAAKC,mBAAmB;IAC1B,IAAI,SAAS,GACX,KAAKZ,IAAI,OACP,yBAAyB,KAAK,IAAI,OAAO,gBAAgB,KAAKa,uBAAuB,GACvF;IAEF,IAAI,CAAC,KAAKL,2BAA2B;KACnC,KAAKA,4BAA4B,iBAAiB;MAChD,KAAKA,4BAA4B,KAAA;MACjC,IAAI,KAAKI,mBACP,KAAKF,oBAAoB;KAE7B,GAAG,KAAKG,sBAAsB;KAC9B,KAAKA,yBAAyB,KAAK,IACjC,KAAKA,yBAAyB,GAC9B,cACF;IACF;GACF;EACF,CAAC;EACH,gBAAgB,EAAC,OAAO,MAAM,OAAM,CAAC,EAAE,GACrC,QACA,KAAKC,qBACP;EACA,KAAKH,gBAAgB;CACvB;CAEA,yBAAkC,SAAiB;EACjD,IAAI;GACF,MAAM,SAAS,MACb,KAAK,MAAM,IAAI,GACf,uBACA,aACF;GACA,MAAM,CAAC,EAAC,cAAc,WAAW,iBAAgB,OAAO;GACxD,MAAM,kBAAmC;IACvC;IACA;IACA,cAAc,OAAO,SAAS,mBAAmB,KAAK,IAAI;GAC5D;GACA,IAAI,gBAAgB,cAAc,KAAKF,kBAAkB,WAAW;IAClE,KAAKT,IAAI,QACP,kCAAkC,KAAKS,kBAAkB,UAAU,MAAM,gBAAgB,aACzF;KAAC,OAAO,KAAKJ;KAAiB,QAAQ;IAAe,CACvD;IACA,KAAKD,QAAQ,KAAK,eAAe;GACnC;GACA,KAAKK,mBAAmB;GAGxB,KAAKI,yBAAyB;EAChC,SAAS,GAAG;GACV,KAAKb,IAAI,QAAQ,qCAAqC,QAAQ,CAAC;EACjE;CACF;CAEA,uBAAuB;EACrB,IAAI,CAAC,KAAKY,mBACR;EAEF,KAAKA,oBAAoB;EACzB,IAAI,KAAKD,iBAAiB,KAAKH,2BAC7B,KAAKR,IAAI,QACP,6DACA;GAAC,OAAO,KAAKK;GAAiB,QAAQ,KAAKI;EAAgB,CAC7D;EAEF,KAAKE,eAAe,KAAK,SAAS;EAClC,KAAKA,gBAAgB,KAAA;EACrB,aAAa,KAAKH,yBAAyB;EAC3C,KAAKA,4BAA4B,KAAA;CACnC;AACF;AAEA,SAAgB,6BAA6B,EAC3C,WACA,UACA,UAKS;CACT,IAAI,aAAa,KAAA,KAAa,WAAW,KAAA,GACvC,OAAO;CAGT,MAAM,MAAM,IAAI,IAAI,SAAS;CAC7B,IAAI,IAAI,aAAa,OACnB,OAAO;CAGT,IAAI,aAAa,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,UAAU,GAC5D,IAAI,aAAa,IAAI,YAAY,QAAQ;CAE3C,IAAI,WAAW,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,QAAQ,GACxD,IAAI,aAAa,IAAI,UAAU,MAAM;CAEvC,OAAO,IAAI,SAAS;AACtB"}

package/out/zero-cache/src/services/http-service.d.ts

@@ -5,6 +5,7 @@ import type { Service } from './service.ts';
 export type Options = {
     port: number;
     keepaliveTimeoutMs: number | undefined;
+    readinessGate?: Promise<void> | undefined;
 };
 /**
  * Common functionality for all HttpServices. These include:

package/out/zero-cache/src/services/http-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;gBAKtC,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAe1D,SAAS,CAAC,QAAQ;IAClB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAgBxB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAKpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAO5B"}
+{"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IAGvC,aAAa,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC;CAC3C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;gBAOtC,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAiB1D,SAAS,CAAC,QAAQ;IAClB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAsBxB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAKpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAO5B"}

package/out/zero-cache/src/services/http-service.js

@@ -16,8 +16,9 @@ var HttpService = class {
 	_state;
 	#heartbeatMonitor;
 	#init;
+	#ready = false;
 	constructor(id, lc, opts, init) {
-		const { port, keepaliveTimeoutMs } = opts;
+		const { port, keepaliveTimeoutMs, readinessGate = promiseVoid } = opts;
 		this.id = id;
 		this._lc = lc.withContext("component", this.id);
 		this.#fastify = Fastify();
@@ -25,16 +26,19 @@ var HttpService = class {
 		this.#init = init;
 		this._state = new RunningState(id);
 		this.#heartbeatMonitor = keepaliveTimeoutMs ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs) : void 0;
+		readinessGate.then(() => this.#ready = true);
 	}
 	_onStart() {}
 	_onStop() {
 		return promiseVoid;
 	}
 	async start() {
-		this.#fastify.get("/", (_req, res) => res.send("OK"));
+		this.#fastify.get("/", (_req, res) => {
+			if (this.#ready) res.send("OK");
+		});
 		this.#fastify.get("/keepalive", ({ headers }, res) => {
 			this.#heartbeatMonitor?.onHeartbeat(headers);
-			return res.send("OK");
+			if (this.#ready) res.send("OK");
 		});
 		await this.#init(this.#fastify);
 		const address = await this.#fastify.listen({

package/out/zero-cache/src/services/http-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.js","names":["#fastify","#port","#heartbeatMonitor","#init"],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => res.send('OK'));\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      return res.send('OK');\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAiBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,uBAAsB;EACnC,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKA,WAAW,QAAQ;EACxB,KAAKC,QAAQ;EACb,KAAKE,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAKD,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;CACN;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAKF,SAAS,IAAI,MAAM,MAAM,QAAQ,IAAI,KAAK,IAAI,CAAC;EACpD,KAAKA,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAKE,mBAAmB,YAAY,OAAO;GAC3C,OAAO,IAAI,KAAK,IAAI;EACtB,CAAC;EACD,MAAM,KAAKC,MAAM,KAAKH,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAKA,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAKC;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAKC,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAKF,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}
+{"version":3,"file":"http-service.js","names":["#fastify","#port","#heartbeatMonitor","#init","#ready"],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n\n  // Wait for the readinessGate to resolve before responding to health checks.\n  readinessGate?: Promise<void> | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  #ready = false;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n\n    void readinessGate.then(() => (this.#ready = true));\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => {\n      if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CAET,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKA,WAAW,QAAQ;EACxB,KAAKC,QAAQ;EACb,KAAKE,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAKD,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAKE,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAKJ,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAKI,QACP,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAKJ,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAKE,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAKE,QACP,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAKD,MAAM,KAAKH,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAKA,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAKC;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAKC,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAKF,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}

package/out/zero-cache/src/services/life-cycle.d.ts

@@ -29,7 +29,7 @@ export declare const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;
 export declare class ProcessManager {
     #private;
     constructor(lc: LogContext, proc: EventEmitter);
-    done(): Promise<void>;
+    shutdown(): Promise<void>;
     addSubprocess(proc: Subprocess, type: WorkerType, name: string): void;
     addWorker(worker: Worker, type: WorkerType, name: string): Worker;
     initializing(): string[];

package/out/zero-cache/src/services/life-cycle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,gCAAiC,CAAC;AAChE,eAAO,MAAM,iBAAiB,iCAAkC,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;gBAWb,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY;IA2C9C,IAAI;IAyBJ,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;IA2B9D,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IA0BjE,YAAY,IAAI,MAAM,EAAE;IAIlB,eAAe;IAIrB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;CAiF3C;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBA6BzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;gBAQf,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM;IAKhD,WAAW,CAAC,UAAU,EAAE,mBAAmB;IAsD3C,IAAI;CAML"}
+{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,gCAAiC,CAAC;AAChE,eAAO,MAAM,iBAAiB,iCAAkC,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;gBAWb,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY;IA2CxC,QAAQ;IA4Bd,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;IA2B9D,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IA0BjE,YAAY,IAAI,MAAM,EAAE;IAIlB,eAAe;IAIrB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;CAiF3C;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBA6BzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;gBAQf,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM;IAKhD,WAAW,CAAC,UAAU,EAAE,mBAAmB;IAsD3C,IAAI;CAML"}

package/out/zero-cache/src/services/life-cycle.js

@@ -63,8 +63,9 @@ var ProcessManager = class {
 			process.exit(code);
 		};
 	}
-	done() {
-		return this.#runningState.stopped();
+	async shutdown() {
+		if (this.#drainStart === 0) this.#startDrain(GRACEFUL_SHUTDOWN[0]);
+		await this.#runningState.stopped();
 	}
 	#exit(code) {
 		this.#lc.info?.("exiting with code", code);
@@ -138,7 +139,7 @@ var ProcessManager = class {
 		this.#lc[log]?.(sig ? `${name} (${pid}) killed with (${sig})` : `${name} (${pid}) exited with code (${code})`, err ?? "");
 		if (this.#userFacing.size === 0) {
 			this.#lc.info?.(this.#drainStart ? `all user-facing workers drained (${Date.now() - this.#drainStart} ms)` : `all user-facing workers exited`);
-			return this.#exit(0);
+			return this.#exit(this.#drainStart ? 0 : code || -1);
 		}
 		if (this.#drainStart === 0) return this.#exit(code || -1);
 	}