@rocicorp/zero 1.0.0 → 1.1.0

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#stream","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  autoReset: boolean,\n  backPressureLimitHeapProportion: number,\n  flowControlConsensusPaddingSeconds: number,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    autoReset,\n    backPressureLimitHeapProportion,\n    flowControlConsensusPaddingSeconds,\n    setTimeoutFn,\n  );\n}\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n */\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    autoReset: boolean,\n    backPressureLimitHeapProportion: number,\n    flowControlConsensusPaddingSeconds: number,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      backPressureLimitHeapProportion,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership();\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        this.#state.resetBackoff();\n        this.#replicationStatusPublisher.publish(\n          this.#lc,\n          'Replicating',\n          `Replicating from ${lastWatermark}`,\n        );\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const entry: WatermarkedChange = [watermark, change];\n          unflushedBytes += this.#storer.store(entry);\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            await this.#forwarder.forwardWithFlowControl(entry);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        await this.#forwarder.forward([\n          watermark,\n          ['rollback', {tag: 'rollback'}],\n        ]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > 5000\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n\n      if (initial) {\n        this.scheduleCleanup(watermark);\n      }\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAsDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,iCACA,oCACA,eAAe,YACiB;AAEhC,OAAM,yBAAyB,IAAI,UAAU,MAAM;AACnD,OAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,aACD;CAED,MAAM,EAAC,mBAAkB;AACzB,QAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,iCACA,oCACA,aACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyJH,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,KAAa;CAU/C,WAAoB,UAAU;CAE9B,aAAsB,mBACpB,eACA,gBACA,mCACD;CAED;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,WACA,iCACA,oCACA,eAAe,YACf;AACA,OAAK,KAAK;AACV,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AACzD,QAAA,QAAc;AACd,QAAA,WAAiB;AACjB,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,SAAe,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,MAAA,QAAc,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;GAAG,CAAC,GACzE,QAAO,KAAK,KAAK,IAAI,EACrB,gCACD;AACD,QAAA,YAAkB,IAAI,UAAU,IAAI,EAClC,oCACD,CAAC;AACF,QAAA,6BAAmC;AACnC,QAAA,YAAkB;AAClB,QAAA,QAAc,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,aAAa;;CAGlE,MAAM,MAAM;AACV,QAAA,GAAS,OAAO,yBAAyB;AAEzC,QAAA,UAAgB,sBAAsB;AAItC,QAAM,MAAA,OAAa,iBAAiB;EAIpC,MAAM,sBAAsB,wBAAwB,MAAM;AAE1D,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;AACrB,OAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,MAAA,OAAa,wCAAwC;IAC7D,MAAM,SAAS,MAAM,MAAA,OAAa,YAChC,eACA,iBACD;AACD,UAAA,OAAa,KAAK,CAAC,OAAM,MAAK,OAAO,QAAQ,OAAO,EAAE,CAAC;AAEvD,UAAA,SAAe;AACf,UAAA,MAAY,cAAc;AAC1B,UAAA,2BAAiC,QAC/B,MAAA,IACA,eACA,oBAAoB,gBACrB;AACD,gBAAY;AAEZ,eAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;AACpB,aAAQ,MAAR;MACE,KAAK;AACH,WAAI,IAAI,IACN,OAAA,OAAa,OAAO,OAAO;AAE7B;MACF,KAAK;AACH,aAAM,MAAA,qBAA2B,IAAI;AACrC;MACF,KAAK;AACH,mBAAY,OAAO,GAAG;AACtB;MACF,KAAK;AACH,WAAI,cAAc,OAAO,GAAG,UAC1B,OAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,YAC7E;AAEH,aAAA,UAAgB,IAAI,EAAE;AACtB;MACF;AACE,WAAI,cAAc,KAChB,OAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,mCAC5B;AAEH;;KAGJ,MAAM,QAA2B,CAAC,WAAW,OAAO;AACpD,uBAAkB,MAAA,OAAa,MAAM,MAAM;AAC3C,SAAI,iBAAiB,oBAEnB,OAAA,UAAgB,QAAQ,MAAM;UACzB;AAQL,YAAM,MAAA,UAAgB,uBAAuB,MAAM;AACnD,uBAAiB;;AAGnB,SAAI,SAAS,YAAY,SAAS,WAChC,aAAY;KAId,MAAM,eAAe,MAAA,OAAa,cAAc;AAChD,SAAI,aACF,OAAM;;YAGH,GAAG;AACV,UAAM;aACE;AACR,UAAA,QAAc,QAAQ,QAAQ;AAC9B,UAAA,SAAe,KAAA;;AAIjB,OAAI,WAAW;AACb,UAAA,GAAS,OAAO,oCAAoC,YAAY;AAChE,UAAA,OAAa,OAAO;AACpB,UAAM,MAAA,UAAgB,QAAQ,CAC5B,WACA,CAAC,YAAY,EAAC,KAAK,YAAW,CAAC,CAChC,CAAC;;AAIJ,SAAM,QAAQ,IAAI;IAChB,MAAA,OAAa,MAAM;IACnB,MAAA,MAAY,QAAQ,MAAA,IAAU,IAAI;IAClC,MAAA,MAAY,aAAa,MACrB,qBACE,MAAA,IACA,uBAAuB,MAAA,IAAU,eAAe,IAAI,CACrD,GACD;IACL,CAAC;;AAGJ,QAAA,UAAgB,qBAAqB;AACrC,QAAA,GAAS,OAAO,yBAAyB;;CAG3C,OAAA,qBAA4B,KAA6B;AACvD,QAAA,GAAS,OAAO,4BAA4B,IAAI;EAChD,MAAM,EAAC,QAAO;AAEd,UAAQ,KAAR;GACE,KAAK;AACH,UAAM,kBAAkB,MAAA,UAAgB,MAAA,MAAY;AACpD,UAAM,wBACJ,MAAA,IACA,eACA,IAAI,WAAW,mBACf,IAAI,aACL;AACD,QAAI,MAAA,WAAiB;AACnB,WAAA,GAAS,OAAO,+BAA+B;AAC/C,WAAM,KAAK,KAAK,IAAI,iBAAiB,CAAC;;AAExC;GACF,QACE,aAAY,IAAI;;;CAItB,UAAU,KAAqD;EAC7D,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,MAAI,SAAS,UACX,OAAA,QAAc,SAAS;EAEzB,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe,MAAA,UAAgB,OAAO,WAAW,EAClD,CAAC;EACF,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,WACD;AACD,MAAI,mBAAmB,MAAA,gBAAsB;AAC3C,SAAA,GAAS,OACP,2CAA2C,iBAC5C;AACD,cAAW,MACT,GACA,8BACE,MAAA,eACD,cAAc,eAAe,GAC/B;SACI;AACL,SAAA,GAAS,QAAQ,qBAAqB,WAAW,KAAK;AAEtD,SAAA,UAAgB,IAAI,WAAW;AAC/B,SAAA,OAAa,QAAQ,YAAY,KAAK;AAEtC,OAAI,QACF,MAAK,gBAAgB,UAAU;;AAGnC,SAAO,QAAQ,QAAQ,WAAW;;CAGpC,gBAAgB,WAAmB;EACjC,MAAM,WAAW,MAAA,kBAAwB;AACzC,QAAA,kBAAwB,IAAI,UAAU;AAEtC,MAAI,aAAa,EACf,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;CAI3E,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,MAAA,OAAa,2BAA2B;AACnE,MAAI,CAAC,aACH,OAAA,GAAS,OACP,+EACD;AAEH,SAAO;GACL,gBAAgB,MAAA;GAChB,cAAc,gBAAgB,MAAA;GAC/B;;;;;;;CAQH,OAAA,kBAAwC;EACtC,MAAM,UAAU,CAAC,GAAG,MAAA,kBAAwB;AAC5C,MAAI,QAAQ,WAAW,GAAG;AACxB,SAAA,GAAS,OAAO,6CAA6C;AAC7D;;EAEF,MAAM,UAAU,CAAC,GAAG,MAAA,UAAgB,SAAS,CAAC;AAC9C,MAAI,QAAQ,WAAW,GAAG;AAGxB,SAAA,GAAS,OAAO,oCAAoC;AACpD;;AAEF,MAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,QAAoC;GACpE,MAAM,kBAAkB,IAAI,GAAI,QAAoC;AACpE,OAAI,kBAAkB,gBACpB,OAAA,GAAS,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,GAC/E;QACI;AACL,UAAA,GAAS,OAAO,0BAA0B,gBAAgB,MAAM;IAChE,MAAM,QAAQ,YAAY,KAAK;IAC/B,MAAM,UAAU,MAAM,MAAA,OAAa,mBAAmB,gBAAgB;IACtE,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,UAAA,GAAS,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,MACjE;AACD,UAAA,kBAAwB,OAAO,gBAAgB;;WAE1C,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;YACtC;AACR,OAAI,MAAA,kBAAwB,KAE1B,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;;CAK7E,MAAM,KAAK,KAAe;AACxB,QAAA,MAAY,KAAK,MAAA,IAAU,IAAI;AAC/B,QAAA,QAAc,QAAQ,QAAQ;AAC9B,QAAM,MAAA,OAAa,MAAM;;;AAmB7B,IAAM,mBAAmB,6BAA6B"}
+{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#stream","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Downstream,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  autoReset: boolean,\n  backPressureLimitHeapProportion: number,\n  flowControlConsensusPaddingSeconds: number,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    autoReset,\n    backPressureLimitHeapProportion,\n    flowControlConsensusPaddingSeconds,\n    setTimeoutFn,\n  );\n}\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n */\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n\n  #latestStatus: Status;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    autoReset: boolean,\n    backPressureLimitHeapProportion: number,\n    flowControlConsensusPaddingSeconds: number,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      backPressureLimitHeapProportion,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership();\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        this.#state.resetBackoff();\n        this.#replicationStatusPublisher.publish(\n          this.#lc,\n          'Replicating',\n          `Replicating from ${lastWatermark}`,\n        );\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const entry: WatermarkedChange = [watermark, change];\n          unflushedBytes += this.#storer.store(entry);\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            await this.#forwarder.forwardWithFlowControl(entry);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, ['rollback', {tag: 'rollback'}]]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > 5000\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n\n      if (initial) {\n        this.scheduleCleanup(watermark);\n      }\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAuDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,iCACA,oCACA,eAAe,YACiB;AAEhC,OAAM,yBAAyB,IAAI,UAAU,MAAM;AACnD,OAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,aACD;CAED,MAAM,EAAC,mBAAkB;AACzB,QAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,iCACA,oCACA,aACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyJH,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,KAAa;CAU/C,WAAoB,UAAU;CAE9B,aAAsB,mBACpB,eACA,gBACA,mCACD;CAED;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,WACA,iCACA,oCACA,eAAe,YACf;AACA,OAAK,KAAK;AACV,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AACzD,QAAA,QAAc;AACd,QAAA,WAAiB;AACjB,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,SAAe,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,MAAA,QAAc,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;GAAG,CAAC,GACzE,QAAO,KAAK,KAAK,IAAI,EACrB,gCACD;AACD,QAAA,YAAkB,IAAI,UAAU,IAAI,EAClC,oCACD,CAAC;AACF,QAAA,6BAAmC;AACnC,QAAA,YAAkB;AAClB,QAAA,QAAc,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,aAAa;AAChE,QAAA,eAAqB,EAAC,KAAK,UAAS;;CAGtC,MAAM,MAAM;AACV,QAAA,GAAS,OAAO,yBAAyB;AAEzC,QAAA,UAAgB,sBAAsB;EAEtC,MAAM,YAAY,MAAM,MAAA,OAAa,kBAAkB;AACvD,MAAI,UACF,OAAA,aAAmB,YAAY;AAKjC,QAAM,MAAA,OAAa,iBAAiB;EAIpC,MAAM,sBAAsB,wBAAwB,MAAM;AAE1D,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;AACrB,OAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,MAAA,OAAa,wCAAwC;IAC7D,MAAM,SAAS,MAAM,MAAA,OAAa,YAChC,eACA,iBACD;AACD,UAAA,OAAa,KAAK,CAAC,OAAM,MAAK,OAAO,QAAQ,OAAO,EAAE,CAAC;AAEvD,UAAA,SAAe;AACf,UAAA,MAAY,cAAc;AAC1B,UAAA,2BAAiC,QAC/B,MAAA,IACA,eACA,oBAAoB,gBACrB;AACD,gBAAY;AAEZ,eAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;AACpB,aAAQ,MAAR;MACE,KAAK;AACH,WAAI,IAAI,IACN,OAAA,OAAa,OAAO,OAAO;AAE7B,WAAI,IAAI,WAAW;AAKjB,cAAA,aAAmB,YAAY,IAAI;AACnC,cAAA,UAAgB,WAAW,MAAA,aAAmB;;AAEhD;MACF,KAAK;AACH,aAAM,MAAA,qBAA2B,IAAI;AACrC;MACF,KAAK;AACH,mBAAY,OAAO,GAAG;AACtB;MACF,KAAK;AACH,WAAI,cAAc,OAAO,GAAG,UAC1B,OAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,YAC7E;AAEH,aAAA,UAAgB,IAAI,EAAE;AACtB;MACF;AACE,WAAI,cAAc,KAChB,OAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,mCAC5B;AAEH;;KAGJ,MAAM,QAA2B,CAAC,WAAW,OAAO;AACpD,uBAAkB,MAAA,OAAa,MAAM,MAAM;AAC3C,SAAI,iBAAiB,oBAEnB,OAAA,UAAgB,QAAQ,MAAM;UACzB;AAQL,YAAM,MAAA,UAAgB,uBAAuB,MAAM;AACnD,uBAAiB;;AAGnB,SAAI,SAAS,YAAY,SAAS,WAChC,aAAY;KAId,MAAM,eAAe,MAAA,OAAa,cAAc;AAChD,SAAI,aACF,OAAM;;YAGH,GAAG;AACV,UAAM;aACE;AACR,UAAA,QAAc,QAAQ,QAAQ;AAC9B,UAAA,SAAe,KAAA;;AAIjB,OAAI,WAAW;AACb,UAAA,GAAS,OAAO,oCAAoC,YAAY;AAChE,UAAA,OAAa,OAAO;AACpB,UAAA,UAAgB,QAAQ,CAAC,WAAW,CAAC,YAAY,EAAC,KAAK,YAAW,CAAC,CAAC,CAAC;;AAIvE,SAAM,QAAQ,IAAI;IAChB,MAAA,OAAa,MAAM;IACnB,MAAA,MAAY,QAAQ,MAAA,IAAU,IAAI;IAClC,MAAA,MAAY,aAAa,MACrB,qBACE,MAAA,IACA,uBAAuB,MAAA,IAAU,eAAe,IAAI,CACrD,GACD;IACL,CAAC;;AAGJ,QAAA,UAAgB,qBAAqB;AACrC,QAAA,GAAS,OAAO,yBAAyB;;CAG3C,OAAA,qBAA4B,KAA6B;AACvD,QAAA,GAAS,OAAO,4BAA4B,IAAI;EAChD,MAAM,EAAC,QAAO;AAEd,UAAQ,KAAR;GACE,KAAK;AACH,UAAM,kBAAkB,MAAA,UAAgB,MAAA,MAAY;AACpD,UAAM,wBACJ,MAAA,IACA,eACA,IAAI,WAAW,mBACf,IAAI,aACL;AACD,QAAI,MAAA,WAAiB;AACnB,WAAA,GAAS,OAAO,+BAA+B;AAC/C,WAAM,KAAK,KAAK,IAAI,iBAAiB,CAAC;;AAExC;GACF,QACE,aAAY,IAAI;;;CAItB,UAAU,KAAqD;EAC7D,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,MAAI,SAAS,UACX,OAAA,QAAc,SAAS;EAEzB,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe,MAAA,UAAgB,OAAO,WAAW,EAClD,CAAC;EACF,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,MAAA,aACP;AACD,MAAI,mBAAmB,MAAA,gBAAsB;AAC3C,SAAA,GAAS,OACP,2CAA2C,iBAC5C;AACD,cAAW,MACT,GACA,8BACE,MAAA,eACD,cAAc,eAAe,GAC/B;SACI;AACL,SAAA,GAAS,QAAQ,qBAAqB,WAAW,KAAK;AAEtD,SAAA,UAAgB,IAAI,WAAW;AAC/B,SAAA,OAAa,QAAQ,YAAY,KAAK;AAEtC,OAAI,QACF,MAAK,gBAAgB,UAAU;;AAGnC,SAAO,QAAQ,QAAQ,WAAW;;CAGpC,gBAAgB,WAAmB;EACjC,MAAM,WAAW,MAAA,kBAAwB;AACzC,QAAA,kBAAwB,IAAI,UAAU;AAEtC,MAAI,aAAa,EACf,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;CAI3E,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,MAAA,OAAa,2BAA2B;AACnE,MAAI,CAAC,aACH,OAAA,GAAS,OACP,+EACD;AAEH,SAAO;GACL,gBAAgB,MAAA;GAChB,cAAc,gBAAgB,MAAA;GAC/B;;;;;;;CAQH,OAAA,kBAAwC;EACtC,MAAM,UAAU,CAAC,GAAG,MAAA,kBAAwB;AAC5C,MAAI,QAAQ,WAAW,GAAG;AACxB,SAAA,GAAS,OAAO,6CAA6C;AAC7D;;EAEF,MAAM,UAAU,CAAC,GAAG,MAAA,UAAgB,SAAS,CAAC;AAC9C,MAAI,QAAQ,WAAW,GAAG;AAGxB,SAAA,GAAS,OAAO,oCAAoC;AACpD;;AAEF,MAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,QAAoC;GACpE,MAAM,kBAAkB,IAAI,GAAI,QAAoC;AACpE,OAAI,kBAAkB,gBACpB,OAAA,GAAS,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,GAC/E;QACI;AACL,UAAA,GAAS,OAAO,0BAA0B,gBAAgB,MAAM;IAChE,MAAM,QAAQ,YAAY,KAAK;IAC/B,MAAM,UAAU,MAAM,MAAA,OAAa,mBAAmB,gBAAgB;IACtE,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,UAAA,GAAS,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,MACjE;AACD,UAAA,kBAAwB,OAAO,gBAAgB;;WAE1C,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;YACtC;AACR,OAAI,MAAA,kBAAwB,KAE1B,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;;CAK7E,MAAM,KAAK,KAAe;AACxB,QAAA,MAAY,KAAK,MAAA,IAAU,IAAI;AAC/B,QAAA,QAAc,QAAQ,QAAQ;AAC9B,QAAM,MAAA,OAAa,MAAM;;;AAmB7B,IAAM,mBAAmB,6BAA6B"}

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts

@@ -100,9 +100,26 @@ export type ChangeEntry = {
  */
 export declare const statusSchema: v.ObjectType<{
     tag: v.Type<"status">;
+    lagReport: v.Optional<{
+        lastTimings?: {
+            sendTimeMs: number;
+            commitTimeMs: number;
+            receiveTimeMs: number;
+        } | undefined;
+        nextSendTimeMs: number;
+    }>;
 }, undefined>;
+export type Status = v.Infer<typeof statusSchema>;
 export declare const statusMessageSchema: v.TupleType<[v.Type<"status">, v.ObjectType<{
     tag: v.Type<"status">;
+    lagReport: v.Optional<{
+        lastTimings?: {
+            sendTimeMs: number;
+            commitTimeMs: number;
+            receiveTimeMs: number;
+        } | undefined;
+        nextSendTimeMs: number;
+    }>;
 }, undefined>]>;
 /**
  * A StatusMessage will be immediately sent on a (v2+) subscription to
@@ -122,6 +139,14 @@ declare const errorSchema: v.TupleType<[v.Type<"error">, v.ObjectType<{
 }, undefined>]>;
 export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status">, v.ObjectType<{
     tag: v.Type<"status">;
+    lagReport: v.Optional<{
+        lastTimings?: {
+            sendTimeMs: number;
+            commitTimeMs: number;
+            receiveTimeMs: number;
+        } | undefined;
+        nextSendTimeMs: number;
+    }>;
 }, undefined>]>, v.UnionType<[v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
     json: v.Optional<"p" | "s">;

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,2CAA2C,CAAC;AAEtE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;CAChE;AAqBD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;aAEvB,CAAC;AAEH,eAAO,MAAM,mBAAmB;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,qBAAsB,SAAQ,cAAc,EAAE,OAAO;IACpE;;;;OAIG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzC,iBAAiB,IAAI,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ"}
+{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,2CAA2C,CAAC;AAEtE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;CAChE;AA0BD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;aASvB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,qBAAsB,SAAQ,cAAc,EAAE,OAAO;IACpE;;;;OAIG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzC,iBAAiB,IAAI,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ"}

package/out/zero-cache/src/services/change-streamer/change-streamer.js

@@ -1,12 +1,19 @@
 import { valita_exports } from "../../../../shared/src/valita.js";
 import "../change-source/protocol/current/data.js";
+import { changeSourceTimingsSchema } from "../replicator/reporter/report-schema.js";
 import { changeStreamDataSchema } from "../change-source/protocol/current/downstream.js";
 //#region ../zero-cache/src/services/change-streamer/change-streamer.ts
 /**
 * The StatusMessage payload for now is empty, but can be extended to
 * include meta-level information in the future.
 */
-var statusSchema = valita_exports.object({ tag: valita_exports.literal("status") });
+var statusSchema = valita_exports.object({
+	tag: valita_exports.literal("status"),
+	lagReport: valita_exports.object({
+		lastTimings: changeSourceTimingsSchema.optional(),
+		nextSendTimeMs: valita_exports.number()
+	}).optional()
+});
 var statusMessageSchema = valita_exports.tuple([valita_exports.literal("status"), statusSchema]);
 var subscriptionErrorSchema = valita_exports.object({
 	type: valita_exports.number(),

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {type Change} from '../change-source/protocol/current/data.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: Adds the \"taskID\" to the subscription context, and support for\n//     the BackupMonitor-mediated \"/snapshot\" request.\n// v4: Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\nexport type ChangeEntry = {\n  change: Change;\n\n  /**\n   * Note that it is technically possible for multiple changes to have\n   * the same watermark, but that of a commit is guaranteed to be final,\n   * so subscribers should only store the watermark of commit changes.\n   */\n  watermark: string;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n});\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService extends ChangeStreamer, Service {\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"mappings":";;;;;;;;AAoIA,IAAa,eAAe,eAAE,OAAO,EACnC,KAAK,eAAE,QAAQ,SAAS,EACzB,CAAC;AAEF,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,SAAS,EAAE,aAAa,CAAC;AAU/E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ,CAAC,UAAU;CAC/B,CAAC;AAIF,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,EAAE,wBAAwB,CAAC;AAE1E,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,YACD"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {type Change} from '../change-source/protocol/current/data.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the BackupMonitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\nexport type ChangeEntry = {\n  change: Change;\n\n  /**\n   * Note that it is technically possible for multiple changes to have\n   * the same watermark, but that of a commit is guaranteed to be final,\n   * so subscribers should only store the watermark of commit changes.\n   */\n  watermark: string;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService extends ChangeStreamer, Service {\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"mappings":";;;;;;;;;AA0IA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,SAAS;CAExB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,UAAU;EACjD,gBAAgB,eAAE,QAAQ;EAC3B,CAAC,CACD,UAAU;CACd,CAAC;AAIF,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,SAAS,EAAE,aAAa,CAAC;AAU/E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ,CAAC,UAAU;CAC/B,CAAC;AAIF,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,EAAE,wBAAwB,CAAC;AAE1E,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,YACD"}

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts

@@ -1,5 +1,6 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { WatermarkedChange } from './change-streamer-service.ts';
+import type { Status } from './change-streamer.ts';
 import type { Subscriber } from './subscriber.ts';
 export type ProgressMonitorOptions = {
     flowControlConsensusPaddingSeconds: number;
@@ -26,6 +27,7 @@ export declare class Forwarder {
      * occasionally to avoid memory blowup.
      */
     forward(entry: WatermarkedChange): void;
+    sendStatus(status: Status): void;
     /**
      * The flow-control-aware equivalent of {@link forward()}, returning a
      * Promise that resolves when replication should continue.

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,sBAAsB,GAAG;IACnC,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF,qBAAa,SAAS;;gBAWlB,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAAgE;IAMxE,oBAAoB;IAsBpB,mBAAmB;IAInB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IAKhC;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB;IA0CrD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}
+{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,sBAAsB,CAAC;AACjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,sBAAsB,GAAG;IACnC,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF,qBAAa,SAAS;;gBAWlB,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAAgE;IAMxE,oBAAoB;IAsBpB,mBAAmB;IAInB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IAKhC,UAAU,CAAC,MAAM,EAAE,MAAM;IAMzB;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB;IA0CrD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}

package/out/zero-cache/src/services/change-streamer/forwarder.js

@@ -53,6 +53,9 @@ var Forwarder = class {
 		Broadcast.withoutTracking(this.#active.values(), entry);
 		this.#updateActiveSubscribers(entry[1]);
 	}
+	sendStatus(status) {
+		for (const sub of this.#active.values()) sub.sendStatus(status);
+	}
 	/**
 	* The flow-control-aware equivalent of {@link forward()}, returning a
 	* Promise that resolves when replication should continue.

package/out/zero-cache/src/services/change-streamer/forwarder.js.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.js","names":["#lc","#progressMonitorOptions","#active","#queued","#progressMonitor","#trackProgress","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport {Broadcast} from './broadcast.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusPaddingSeconds: number;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #progressMonitorOptions: ProgressMonitorOptions;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {flowControlConsensusPaddingSeconds: 1},\n  ) {\n    this.#lc = lc.withContext('component', 'progress-monitor');\n    this.#progressMonitorOptions = opts;\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(this.#trackProgress, 1000);\n  }\n\n  readonly #trackProgress = () => {\n    const now = performance.now();\n    for (const sub of this.#active) {\n      sub.sampleProcessRate(now);\n    }\n\n    const {flowControlConsensusPaddingSeconds} = this.#progressMonitorOptions;\n    // A negative number disables early flow control release.\n    if (flowControlConsensusPaddingSeconds >= 0) {\n      this.#currentBroadcast?.checkProgress(\n        this.#lc,\n        flowControlConsensusPaddingSeconds * 1000,\n        now,\n      );\n    }\n  };\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const broadcast = new Broadcast(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    await broadcast.done;\n\n    // Technically #currentBroadcast may have changed, so only\n    // unset if it if is still the same.\n    if (this.#currentBroadcast === broadcast) {\n      this.#currentBroadcast = undefined;\n    }\n  }\n\n  #updateActiveSubscribers([type]: ChangeStreamData) {\n    switch (type) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;AAWA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA,0BAAmB,IAAI,KAAiB;CACxC,0BAAmB,IAAI,KAAiB;CACxC,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAAC,oCAAoC,GAAE,EACtE;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,mBAAmB;AAC1D,QAAA,yBAA+B;;CAGjC,uBAAuB;AACrB,gBAAc,MAAA,gBAAsB;AACpC,QAAA,kBAAwB,YAAY,MAAA,eAAqB,IAAK;;CAGhE,uBAAgC;EAC9B,MAAM,MAAM,YAAY,KAAK;AAC7B,OAAK,MAAM,OAAO,MAAA,OAChB,KAAI,kBAAkB,IAAI;EAG5B,MAAM,EAAC,uCAAsC,MAAA;AAE7C,MAAI,sCAAsC,EACxC,OAAA,kBAAwB,cACtB,MAAA,IACA,qCAAqC,KACrC,IACD;;CAIL,sBAAsB;AACpB,gBAAc,MAAA,gBAAsB;;;;;;;CAQtC,IAAI,KAAiB;AACnB,MAAI,MAAA,cACF,OAAA,OAAa,IAAI,IAAI;MAErB,OAAA,OAAa,IAAI,IAAI;;CAIzB,OAAO,KAAiB;AACtB,QAAA,OAAa,OAAO,IAAI;AACxB,QAAA,OAAa,OAAO,IAAI;AACxB,MAAI,OAAO;;;;;;;;;;;CAYb,QAAQ,OAA0B;AAChC,YAAU,gBAAgB,MAAA,OAAa,QAAQ,EAAE,MAAM;AACvD,QAAA,wBAA8B,MAAM,GAAG;;;;;;CAOzC,MAAM,uBAAuB,OAA0B;EACrD,MAAM,YAAY,IAAI,UAAU,MAAA,OAAa,QAAQ,EAAE,MAAM;AAC7D,QAAA,wBAA8B,MAAM,GAAG;AAGvC,QAAA,mBAAyB;AAEzB,QAAM,UAAU;AAIhB,MAAI,MAAA,qBAA2B,UAC7B,OAAA,mBAAyB,KAAA;;CAI7B,yBAAyB,CAAC,OAAyB;AACjD,UAAQ,MAAR;GACE,KAAK;AAKH,UAAA,gBAAsB;AACtB;GACF,KAAK;GACL,KAAK;AAOH,UAAA,gBAAsB;AACtB,SAAK,MAAM,OAAO,MAAA,OAAa,QAAQ,CACrC,OAAA,OAAa,IAAI,IAAI;AAEvB,UAAA,OAAa,OAAO;AACpB;;;CAIN,UAAuB;AACrB,SAAO,IAAI,IACT,cACE,aAAa,MAAA,OAAa,CAAC,KAAI,MAAK,EAAE,MAAM,EAC5C,aAAa,MAAA,OAAa,CAAC,KAAI,MAAK,EAAE,MAAM,CAC7C,CACF"}
+{"version":3,"file":"forwarder.js","names":["#lc","#progressMonitorOptions","#active","#queued","#progressMonitor","#trackProgress","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport {Broadcast} from './broadcast.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Status} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusPaddingSeconds: number;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #progressMonitorOptions: ProgressMonitorOptions;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {flowControlConsensusPaddingSeconds: 1},\n  ) {\n    this.#lc = lc.withContext('component', 'progress-monitor');\n    this.#progressMonitorOptions = opts;\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(this.#trackProgress, 1000);\n  }\n\n  readonly #trackProgress = () => {\n    const now = performance.now();\n    for (const sub of this.#active) {\n      sub.sampleProcessRate(now);\n    }\n\n    const {flowControlConsensusPaddingSeconds} = this.#progressMonitorOptions;\n    // A negative number disables early flow control release.\n    if (flowControlConsensusPaddingSeconds >= 0) {\n      this.#currentBroadcast?.checkProgress(\n        this.#lc,\n        flowControlConsensusPaddingSeconds * 1000,\n        now,\n      );\n    }\n  };\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const broadcast = new Broadcast(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    await broadcast.done;\n\n    // Technically #currentBroadcast may have changed, so only\n    // unset if it if is still the same.\n    if (this.#currentBroadcast === broadcast) {\n      this.#currentBroadcast = undefined;\n    }\n  }\n\n  #updateActiveSubscribers([type]: ChangeStreamData) {\n    switch (type) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;AAYA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA,0BAAmB,IAAI,KAAiB;CACxC,0BAAmB,IAAI,KAAiB;CACxC,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAAC,oCAAoC,GAAE,EACtE;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,mBAAmB;AAC1D,QAAA,yBAA+B;;CAGjC,uBAAuB;AACrB,gBAAc,MAAA,gBAAsB;AACpC,QAAA,kBAAwB,YAAY,MAAA,eAAqB,IAAK;;CAGhE,uBAAgC;EAC9B,MAAM,MAAM,YAAY,KAAK;AAC7B,OAAK,MAAM,OAAO,MAAA,OAChB,KAAI,kBAAkB,IAAI;EAG5B,MAAM,EAAC,uCAAsC,MAAA;AAE7C,MAAI,sCAAsC,EACxC,OAAA,kBAAwB,cACtB,MAAA,IACA,qCAAqC,KACrC,IACD;;CAIL,sBAAsB;AACpB,gBAAc,MAAA,gBAAsB;;;;;;;CAQtC,IAAI,KAAiB;AACnB,MAAI,MAAA,cACF,OAAA,OAAa,IAAI,IAAI;MAErB,OAAA,OAAa,IAAI,IAAI;;CAIzB,OAAO,KAAiB;AACtB,QAAA,OAAa,OAAO,IAAI;AACxB,QAAA,OAAa,OAAO,IAAI;AACxB,MAAI,OAAO;;;;;;;;;;;CAYb,QAAQ,OAA0B;AAChC,YAAU,gBAAgB,MAAA,OAAa,QAAQ,EAAE,MAAM;AACvD,QAAA,wBAA8B,MAAM,GAAG;;CAGzC,WAAW,QAAgB;AACzB,OAAK,MAAM,OAAO,MAAA,OAAa,QAAQ,CACrC,KAAI,WAAW,OAAO;;;;;;CAQ1B,MAAM,uBAAuB,OAA0B;EACrD,MAAM,YAAY,IAAI,UAAU,MAAA,OAAa,QAAQ,EAAE,MAAM;AAC7D,QAAA,wBAA8B,MAAM,GAAG;AAGvC,QAAA,mBAAyB;AAEzB,QAAM,UAAU;AAIhB,MAAI,MAAA,qBAA2B,UAC7B,OAAA,mBAAyB,KAAA;;CAI7B,yBAAyB,CAAC,OAAyB;AACjD,UAAQ,MAAR;GACE,KAAK;AAKH,UAAA,gBAAsB;AACtB;GACF,KAAK;GACL,KAAK;AAOH,UAAA,gBAAsB;AACtB,SAAK,MAAM,OAAO,MAAA,OAAa,QAAQ,CACrC,OAAA,OAAa,IAAI,IAAI;AAEvB,UAAA,OAAa,OAAO;AACpB;;;CAIN,UAAuB;AACrB,SAAO,IAAI,IACT,cACE,aAAa,MAAA,OAAa,CAAC,KAAI,MAAK,EAAE,MAAM,EAC5C,aAAa,MAAA,OAAa,CAAC,KAAI,MAAK,EAAE,MAAM,CAC7C,CACF"}

package/out/zero-cache/src/services/change-streamer/subscriber.d.ts

@@ -2,7 +2,7 @@ import type { Enum } from '../../../../shared/src/enum.ts';
 import type { Subscription } from '../../types/subscription.ts';
 import type { ChangeStreamData } from '../change-source/protocol/current.ts';
 import type { WatermarkedChange } from './change-streamer-service.ts';
-import { type Downstream } from './change-streamer.ts';
+import { type Downstream, type Status } from './change-streamer.ts';
 import * as ErrorType from './error-type-enum.ts';
 type ErrorType = Enum<typeof ErrorType>;
 /**
@@ -15,10 +15,11 @@ type ErrorType = Enum<typeof ErrorType>;
 export declare class Subscriber {
     #private;
     readonly id: string;
-    constructor(protocolVersion: number, id: string, watermark: string, downstream: Subscription<Downstream>);
+    constructor(protocolVersion: number, id: string, watermark: string, downstream: Subscription<Downstream>, latestStatus: () => Status);
     get watermark(): string;
     get acked(): string;
     send(change: WatermarkedChange): Promise<void>;
+    sendStatus(status: Status): void;
     /** catchup() is called on ChangeEntries loaded from the store. */
     catchup(change: WatermarkedChange): Promise<void>;
     /**

package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAGzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,sCAAsC,CAAC;AAC3E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAElD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC;;;;;;GAMG;AACH,qBAAa,UAAU;;IAErB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAOlB,eAAe,EAAE,MAAM,EACvB,EAAE,EAAE,MAAM,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,YAAY,CAAC,UAAU,CAAC;IAUtC,IAAI,SAAS,WAEZ;IAED,IAAI,KAAK,WAER;IAEK,IAAI,CAAC,MAAM,EAAE,iBAAiB;IAoBpC,kEAAkE;IAC5D,OAAO,CAAC,MAAM,EAAE,iBAAiB;IAKvC;;;OAGG;IACH,WAAW;IA0DX;;OAEG;IACH,IAAI,UAAU,WAEb;IAED;;;OAGG;IACH,IAAI,YAAY,WAEf;IAED;;;OAGG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,SAAK,GAAG,IAAI;IAQrD,QAAQ,IAAI;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAC;IAalD,eAAe,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAS3C,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO;IAIlB,KAAK,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM;CAS1C"}
+{"version":3,"file":"subscriber.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAGzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,sCAAsC,CAAC;AAC3E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAC,KAAK,UAAU,EAAE,KAAK,MAAM,EAAC,MAAM,sBAAsB,CAAC;AAClE,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAElD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC;;;;;;GAMG;AACH,qBAAa,UAAU;;IAErB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAQlB,eAAe,EAAE,MAAM,EACvB,EAAE,EAAE,MAAM,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,YAAY,CAAC,UAAU,CAAC,EACpC,YAAY,EAAE,MAAM,MAAM;IAW5B,IAAI,SAAS,WAEZ;IAED,IAAI,KAAK,WAER;IAEK,IAAI,CAAC,MAAM,EAAE,iBAAiB;IAwBpC,UAAU,CAAC,MAAM,EAAE,MAAM;IAMzB,kEAAkE;IAC5D,OAAO,CAAC,MAAM,EAAE,iBAAiB;IAKvC;;;OAGG;IACH,WAAW;IA0DX;;OAEG;IACH,IAAI,UAAU,WAEb;IAED;;;OAGG;IACH,IAAI,YAAY,WAEf;IAED;;;OAGG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,SAAK,GAAG,IAAI;IAQrD,QAAQ,IAAI;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAC;IAalD,eAAe,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAS3C,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO;IAIlB,KAAK,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM;CAS1C"}

package/out/zero-cache/src/services/change-streamer/subscriber.js

@@ -14,13 +14,15 @@ var Subscriber = class {
 	#protocolVersion;
 	id;
 	#downstream;
+	#latestStatus;
 	#watermark;
 	#acked;
 	#backlog;
-	constructor(protocolVersion, id, watermark, downstream) {
+	constructor(protocolVersion, id, watermark, downstream, latestStatus) {
 		this.#protocolVersion = protocolVersion;
 		this.id = id;
 		this.#downstream = downstream;
+		this.#latestStatus = latestStatus;
 		this.#watermark = watermark;
 		this.#acked = watermark;
 		this.#backlog = [];
@@ -36,16 +38,23 @@ var Subscriber = class {
 		if (watermark > this.#watermark) if (this.#backlog) this.#backlog.push(change);
 		else await this.#sendChange(change);
 	}
-	#initialStatusSent = false;
-	#ensureInitialStatusSent() {
-		if (this.#protocolVersion >= 2 && !this.#initialStatusSent) {
-			this.#sendDownstream(["status", { tag: "status" }]);
-			this.#initialStatusSent = true;
+	#initialized = false;
+	/**
+	* Called once the subscriber's watermark has been validated in the initial
+	* catchup process.
+	*/
+	#initialize() {
+		if (!this.#initialized) {
+			this.#initialized = true;
+			this.sendStatus(this.#latestStatus());
 		}
 	}
+	sendStatus(status) {
+		if (this.#protocolVersion >= 2 && this.#initialized) this.#sendDownstream(["status", status]);
+	}
 	/** catchup() is called on ChangeEntries loaded from the store. */
 	async catchup(change) {
-		this.#ensureInitialStatusSent();
+		this.#initialize();
 		await this.#sendChange(change);
 	}
 	/**
@@ -53,7 +62,7 @@ var Subscriber = class {
 	* entries that were received during the catchup.
 	*/
 	setCaughtUp() {
-		this.#ensureInitialStatusSent();
+		this.#initialize();
 		assert(this.#backlog, "setCaughtUp() called but subscriber is not in catchup mode");
 		for (const change of this.#backlog) this.#sendChange(change);
 		this.#backlog = null;

package/out/zero-cache/src/services/change-streamer/subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#watermark","#acked","#backlog","#sendChange","#initialStatusSent","#sendDownstream","#ensureInitialStatusSent","#pending","#processed","#samples"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type Downstream} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly #downstream: Subscription<Downstream>;\n  #watermark: string;\n  #acked: string;\n  #backlog: WatermarkedChange[] | null;\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    watermark: string,\n    downstream: Subscription<Downstream>,\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.#downstream = downstream;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = [];\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  async send(change: WatermarkedChange) {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        this.#backlog.push(change);\n      } else {\n        await this.#sendChange(change);\n      }\n    }\n  }\n\n  #initialStatusSent = false;\n\n  #ensureInitialStatusSent() {\n    if (this.#protocolVersion >= 2 && !this.#initialStatusSent) {\n      void this.#sendDownstream(['status', {tag: 'status'}]);\n      this.#initialStatusSent = true;\n    }\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#ensureInitialStatusSent();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp() {\n    this.#ensureInitialStatusSent();\n    assert(\n      this.#backlog,\n      'setCaughtUp() called but subscriber is not in catchup mode',\n    );\n    // Note that this method must be asynchronous in order for send() to\n    // interpret the #backlog variable correctly. This is the only place\n    // where I/O flow control is not heeded. However, it will be awaited\n    // by the next caller to send().\n    for (const change of this.#backlog) {\n      void this.#sendChange(change);\n    }\n    this.#backlog = null;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, downstream] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(downstream[1])) {\n      return;\n    }\n    if (downstream[0] === 'commit') {\n      this.#watermark = watermark;\n    }\n    const result = await this.#sendDownstream(downstream);\n    if (downstream[0] === 'commit' && result === 'consumed') {\n      this.#acked = max(this.#acked, watermark);\n    }\n  }\n\n  async #sendDownstream(downstream: Downstream) {\n    this.#pending++;\n    const {result} = this.#downstream.push(downstream);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#processed++;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): {processRate: number; pending: number} {\n    const pending = this.#pending;\n    if (this.#samples.length < 2) {\n      return {processRate: 0, pending};\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {processRate, pending};\n  }\n\n  supportsMessage(change: ChangeStreamData[1]) {\n    switch (change.tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  fail(err?: unknown) {\n    this.close(ErrorType.Unknown, String(err));\n  }\n\n  close(error?: ErrorType, message?: string) {\n    if (error) {\n      const {result} = this.#downstream.push(['error', {type: error, message}]);\n      // Wait for the ACK of the error message before closing the connection.\n      void result.then(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAmBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,iBACA,IACA,WACA,YACA;AACA,QAAA,kBAAwB;AACxB,OAAK,KAAK;AACV,QAAA,aAAmB;AACnB,QAAA,YAAkB;AAClB,QAAA,QAAc;AACd,QAAA,UAAgB,EAAE;;CAGpB,IAAI,YAAY;AACd,SAAO,MAAA;;CAGT,IAAI,QAAQ;AACV,SAAO,MAAA;;CAGT,MAAM,KAAK,QAA2B;EACpC,MAAM,CAAC,aAAa;AACpB,MAAI,YAAY,MAAA,UACd,KAAI,MAAA,QACF,OAAA,QAAc,KAAK,OAAO;MAE1B,OAAM,MAAA,WAAiB,OAAO;;CAKpC,qBAAqB;CAErB,2BAA2B;AACzB,MAAI,MAAA,mBAAyB,KAAK,CAAC,MAAA,mBAAyB;AACrD,SAAA,eAAqB,CAAC,UAAU,EAAC,KAAK,UAAS,CAAC,CAAC;AACtD,SAAA,oBAA0B;;;;CAK9B,MAAM,QAAQ,QAA2B;AACvC,QAAA,yBAA+B;AAC/B,QAAM,MAAA,WAAiB,OAAO;;;;;;CAOhC,cAAc;AACZ,QAAA,yBAA+B;AAC/B,SACE,MAAA,SACA,6DACD;AAKD,OAAK,MAAM,UAAU,MAAA,QACd,OAAA,WAAiB,OAAO;AAE/B,QAAA,UAAgB;;CAGlB,OAAA,WAAkB,QAA2B;EAC3C,MAAM,CAAC,WAAW,cAAc;AAChC,MAAI,aAAa,KAAK,UACpB;AAEF,MAAI,CAAC,KAAK,gBAAgB,WAAW,GAAG,CACtC;AAEF,MAAI,WAAW,OAAO,SACpB,OAAA,YAAkB;EAEpB,MAAM,SAAS,MAAM,MAAA,eAAqB,WAAW;AACrD,MAAI,WAAW,OAAO,YAAY,WAAW,WAC3C,OAAA,QAAc,IAAI,MAAA,OAAa,UAAU;;CAI7C,OAAA,eAAsB,YAAwB;AAC5C,QAAA;EACA,MAAM,EAAC,WAAU,MAAA,WAAiB,KAAK,WAAW;AAClD,MAAI;AACF,UAAO,MAAM;YACL;AACR,SAAA;AACA,SAAA;;;CAYJ,WAAW;CACX,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,KAAK;EAAC,CAC7C;;;;CAKD,IAAI,aAAa;AACf,SAAO,MAAA;;;;;;CAOT,IAAI,eAAe;AACjB,SAAO,MAAA;;;;;;CAOT,kBAAkB,KAAa,aAAa,IAAU;AACpD,SAAO,MAAA,QAAc,UAAU,WAC7B,OAAA,QAAc,OAAO;AAEvB,QAAA,QAAc,KAAK;GAAC,WAAW,MAAA;GAAiB,WAAW;GAAI,CAAC;AAChE,SAAO;;CAGT,WAAmD;EACjD,MAAM,UAAU,MAAA;AAChB,MAAI,MAAA,QAAc,SAAS,EACzB,QAAO;GAAC,aAAa;GAAG;GAAQ;EAElC,MAAM,OAAO,MAAA,QAAc;EAC3B,MAAM,KAAK,KAAK,MAAA,QAAc,GAAG,GAAG,CAAC;EACrC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;AAElD,SAAO;GAAC,aADY,YAAY,IAAI,IAAI,YAAY;GAC/B;GAAQ;;CAG/B,gBAAgB,QAA6B;AAC3C,UAAQ,OAAO,KAAf;GACE,KAAK,wBAEH,QAAO,MAAA,mBAAyB;;AAEpC,SAAO;;CAGT,KAAK,KAAe;AAClB,OAAK,MAAM,GAAmB,OAAO,IAAI,CAAC;;CAG5C,MAAM,OAAmB,SAAkB;AACzC,MAAI,OAAO;GACT,MAAM,EAAC,WAAU,MAAA,WAAiB,KAAK,CAAC,SAAS;IAAC,MAAM;IAAO;IAAQ,CAAC,CAAC;AAEpE,UAAO,WAAW,MAAA,WAAiB,QAAQ,CAAC;QAEjD,OAAA,WAAiB,QAAQ"}
+{"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#latestStatus","#watermark","#acked","#backlog","#sendChange","#initialized","#sendDownstream","#initialize","#pending","#processed","#samples"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type Downstream, type Status} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly #downstream: Subscription<Downstream>;\n  readonly #latestStatus: () => Status;\n  #watermark: string;\n  #acked: string;\n  #backlog: WatermarkedChange[] | null;\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    watermark: string,\n    downstream: Subscription<Downstream>,\n    latestStatus: () => Status,\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.#downstream = downstream;\n    this.#latestStatus = latestStatus;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = [];\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  async send(change: WatermarkedChange) {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        this.#backlog.push(change);\n      } else {\n        await this.#sendChange(change);\n      }\n    }\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      this.sendStatus(this.#latestStatus());\n    }\n  }\n\n  sendStatus(status: Status) {\n    if (this.#protocolVersion >= 2 && this.#initialized) {\n      void this.#sendDownstream(['status', status]);\n    }\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp() {\n    this.#initialize();\n    assert(\n      this.#backlog,\n      'setCaughtUp() called but subscriber is not in catchup mode',\n    );\n    // Note that this method must be asynchronous in order for send() to\n    // interpret the #backlog variable correctly. This is the only place\n    // where I/O flow control is not heeded. However, it will be awaited\n    // by the next caller to send().\n    for (const change of this.#backlog) {\n      void this.#sendChange(change);\n    }\n    this.#backlog = null;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, downstream] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(downstream[1])) {\n      return;\n    }\n    if (downstream[0] === 'commit') {\n      this.#watermark = watermark;\n    }\n    const result = await this.#sendDownstream(downstream);\n    if (downstream[0] === 'commit' && result === 'consumed') {\n      this.#acked = max(this.#acked, watermark);\n    }\n  }\n\n  async #sendDownstream(downstream: Downstream) {\n    this.#pending++;\n    const {result} = this.#downstream.push(downstream);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#processed++;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): {processRate: number; pending: number} {\n    const pending = this.#pending;\n    if (this.#samples.length < 2) {\n      return {processRate: 0, pending};\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {processRate, pending};\n  }\n\n  supportsMessage(change: ChangeStreamData[1]) {\n    switch (change.tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  fail(err?: unknown) {\n    this.close(ErrorType.Unknown, String(err));\n  }\n\n  close(error?: ErrorType, message?: string) {\n    if (error) {\n      const {result} = this.#downstream.push(['error', {type: error, message}]);\n      // Wait for the ACK of the error message before closing the connection.\n      void result.then(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAmBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,iBACA,IACA,WACA,YACA,cACA;AACA,QAAA,kBAAwB;AACxB,OAAK,KAAK;AACV,QAAA,aAAmB;AACnB,QAAA,eAAqB;AACrB,QAAA,YAAkB;AAClB,QAAA,QAAc;AACd,QAAA,UAAgB,EAAE;;CAGpB,IAAI,YAAY;AACd,SAAO,MAAA;;CAGT,IAAI,QAAQ;AACV,SAAO,MAAA;;CAGT,MAAM,KAAK,QAA2B;EACpC,MAAM,CAAC,aAAa;AACpB,MAAI,YAAY,MAAA,UACd,KAAI,MAAA,QACF,OAAA,QAAc,KAAK,OAAO;MAE1B,OAAM,MAAA,WAAiB,OAAO;;CAKpC,eAAe;;;;;CAMf,cAAc;AACZ,MAAI,CAAC,MAAA,aAAmB;AACtB,SAAA,cAAoB;AACpB,QAAK,WAAW,MAAA,cAAoB,CAAC;;;CAIzC,WAAW,QAAgB;AACzB,MAAI,MAAA,mBAAyB,KAAK,MAAA,YAC3B,OAAA,eAAqB,CAAC,UAAU,OAAO,CAAC;;;CAKjD,MAAM,QAAQ,QAA2B;AACvC,QAAA,YAAkB;AAClB,QAAM,MAAA,WAAiB,OAAO;;;;;;CAOhC,cAAc;AACZ,QAAA,YAAkB;AAClB,SACE,MAAA,SACA,6DACD;AAKD,OAAK,MAAM,UAAU,MAAA,QACd,OAAA,WAAiB,OAAO;AAE/B,QAAA,UAAgB;;CAGlB,OAAA,WAAkB,QAA2B;EAC3C,MAAM,CAAC,WAAW,cAAc;AAChC,MAAI,aAAa,KAAK,UACpB;AAEF,MAAI,CAAC,KAAK,gBAAgB,WAAW,GAAG,CACtC;AAEF,MAAI,WAAW,OAAO,SACpB,OAAA,YAAkB;EAEpB,MAAM,SAAS,MAAM,MAAA,eAAqB,WAAW;AACrD,MAAI,WAAW,OAAO,YAAY,WAAW,WAC3C,OAAA,QAAc,IAAI,MAAA,OAAa,UAAU;;CAI7C,OAAA,eAAsB,YAAwB;AAC5C,QAAA;EACA,MAAM,EAAC,WAAU,MAAA,WAAiB,KAAK,WAAW;AAClD,MAAI;AACF,UAAO,MAAM;YACL;AACR,SAAA;AACA,SAAA;;;CAYJ,WAAW;CACX,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,KAAK;EAAC,CAC7C;;;;CAKD,IAAI,aAAa;AACf,SAAO,MAAA;;;;;;CAOT,IAAI,eAAe;AACjB,SAAO,MAAA;;;;;;CAOT,kBAAkB,KAAa,aAAa,IAAU;AACpD,SAAO,MAAA,QAAc,UAAU,WAC7B,OAAA,QAAc,OAAO;AAEvB,QAAA,QAAc,KAAK;GAAC,WAAW,MAAA;GAAiB,WAAW;GAAI,CAAC;AAChE,SAAO;;CAGT,WAAmD;EACjD,MAAM,UAAU,MAAA;AAChB,MAAI,MAAA,QAAc,SAAS,EACzB,QAAO;GAAC,aAAa;GAAG;GAAQ;EAElC,MAAM,OAAO,MAAA,QAAc;EAC3B,MAAM,KAAK,KAAK,MAAA,QAAc,GAAG,GAAG,CAAC;EACrC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;AAElD,SAAO;GAAC,aADY,YAAY,IAAI,IAAI,YAAY;GAC/B;GAAQ;;CAG/B,gBAAgB,QAA6B;AAC3C,UAAQ,OAAO,KAAf;GACE,KAAK,wBAEH,QAAO,MAAA,mBAAyB;;AAEpC,SAAO;;CAGT,KAAK,KAAe;AAClB,OAAK,MAAM,GAAmB,OAAO,IAAI,CAAC;;CAG5C,MAAM,OAAmB,SAAkB;AACzC,MAAI,OAAO;GACT,MAAM,EAAC,WAAU,MAAA,WAAiB,KAAK,CAAC,SAAS;IAAC,MAAM;IAAO;IAAQ,CAAC,CAAC;AAEpE,UAAO,WAAW,MAAA,WAAiB,QAAQ,CAAC;QAEjD,OAAA,WAAiB,QAAQ"}

package/out/zero-cache/src/services/life-cycle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,gCAAiC,CAAC;AAChE,eAAO,MAAM,iBAAiB,sBAAuB,CAAC;AAEtD;;;GAGG;AACH,qBAAa,cAAc;;gBAWb,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY;IAiC9C,IAAI;IAoBJ,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;IAiB9D,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAiBjE,YAAY,IAAI,MAAM,EAAE;IAIlB,eAAe;IAIrB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;CAqE3C;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAWvD;AAID;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;gBAQf,EAAE,EAAE,UAAU,EAAE,YAAY,SAA2B;IAKnE,WAAW,CAAC,UAAU,EAAE,mBAAmB;IAsD3C,IAAI;CAML"}
+{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,gCAAiC,CAAC;AAChE,eAAO,MAAM,iBAAiB,sBAAuB,CAAC;AAEtD;;;GAGG;AACH,qBAAa,cAAc;;gBAWb,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY;IAiC9C,IAAI;IAoBJ,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;IA2B9D,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAiBjE,YAAY,IAAI,MAAM,EAAE;IAIlB,eAAe;IAIrB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;CAqE3C;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAWvD;AAID;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;gBAQf,EAAE,EAAE,UAAU,EAAE,YAAY,SAA2B;IAKnE,WAAW,CAAC,UAAU,EAAE,mBAAmB;IAsD3C,IAAI;CAML"}

package/out/zero-cache/src/services/life-cycle.js

@@ -45,8 +45,12 @@ var ProcessManager = class {
 	addSubprocess(proc, type, name) {
 		if (type === "user-facing") this.#userFacing.add(proc);
 		this.#all.add(proc);
-		proc.on("error", (err) => this.#lc.error?.(`error from ${name} ${proc.pid}`, err));
-		proc.on("close", (code, signal) => this.#onExit(code, signal, null, type, name, proc));
+		let isOpen = true;
+		proc.on("close", (code, signal) => {
+			isOpen = false;
+			this.#onExit(code, signal, null, type, name, proc);
+		});
+		proc.on("error", (err) => this.#lc[isOpen ? "error" : "warn"]?.(`error from ${name} ${proc.pid}`, err));
 	}
 	#initializing = /* @__PURE__ */ new Map();
 	#nextID = 0;

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT`, all workers\n * are stopped without draining. Additionally, if any worker exits\n * unexpectedly, all workers sent an immediate `SIGQUIT` signal.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT'] as const;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0 ? GRACEFUL_SHUTDOWN[0] : FORCEFUL_SHUTDOWN[0],\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` to all workers. For this signal, workers are\n    // stopped immediately without draining. See `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#exit(-1));\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: 'SIGTERM' | 'SIGINT' = 'SIGTERM') {\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    proc.on('error', err =>\n      this.#lc.error?.(`error from ${name} ${proc.pid}`, err),\n    );\n    proc.on('close', (code, signal) =>\n      this.#onExit(code, signal, null, type, name, proc),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown | null,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // The replication-manager has no user-facing workers.\n      // In this case, code === 0 shutdowns are not errors.\n      // Non-zero exits are warnings (not errors) since they're often transient issues.\n      const log = code === 0 && this.#userFacing.size === 0 ? 'info' : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(log === 'info' ? code : -1);\n    }\n\n    const log = this.#drainStart === 0 ? 'error' : 'warn';\n    if (sig) {\n      this.#lc[log]?.(`${name} (${pid}) killed with (${sig})`, err ?? '');\n    } else if (code !== 0) {\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n    } else {\n      this.#lc.info?.(`${name} (${pid}) exited with code (${code})`);\n    }\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    // Exit only if not draining. If a user-facing worker exits unexpectedly\n    // during a drain, log a warning but let other user-facing workers drain.\n    if (log === 'error') {\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n  try {\n    await run();\n    // oxlint-disable-next-line no-console\n    console.info(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    // oxlint-disable-next-line no-console\n    console.error(`pid ${pid} exiting with error`, e);\n    process.exit(-1);\n  }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;AA2BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,UAAU;;;;;AAM5C,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IAAI,kBAAkB,KAAK,kBAAkB,GACvD,CACF;AAKD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,KAAW,GAAG,CAAC;AAGvC,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,SAA+B,WAAW;AACpD,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;AAEnB,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,QAAQ,cAAc,KAAK,GAAG,KAAK,OAAO,IAAI,CACxD;AACD,OAAK,GAAG,UAAU,MAAM,WACtB,MAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK,CACnD;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;GAIzB,MAAM,MAAM,SAAS,KAAK,MAAA,WAAiB,SAAS,IAAI,SAAS;AACjE,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,QAAQ,SAAS,OAAO,GAAG;;EAG/C,MAAM,MAAM,MAAA,eAAqB,IAAI,UAAU;AAC/C,MAAI,IACF,OAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,IAAI,OAAO,GAAG;WAC1D,SAAS,EAClB,OAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;MAEzE,OAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,GAAG;AAIhE,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAKtB,MAAI,QAAQ,QACV,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;AAIpB,IAAM,2BAA2B;;;;;;;;;;;;AAajC,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}
+{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT`, all workers\n * are stopped without draining. Additionally, if any worker exits\n * unexpectedly, all workers sent an immediate `SIGQUIT` signal.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT'] as const;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0 ? GRACEFUL_SHUTDOWN[0] : FORCEFUL_SHUTDOWN[0],\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` to all workers. For this signal, workers are\n    // stopped immediately without draining. See `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#exit(-1));\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: 'SIGTERM' | 'SIGINT' = 'SIGTERM') {\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[isOpen ? 'error' : 'warn']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown | null,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // The replication-manager has no user-facing workers.\n      // In this case, code === 0 shutdowns are not errors.\n      // Non-zero exits are warnings (not errors) since they're often transient issues.\n      const log = code === 0 && this.#userFacing.size === 0 ? 'info' : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(log === 'info' ? code : -1);\n    }\n\n    const log = this.#drainStart === 0 ? 'error' : 'warn';\n    if (sig) {\n      this.#lc[log]?.(`${name} (${pid}) killed with (${sig})`, err ?? '');\n    } else if (code !== 0) {\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n    } else {\n      this.#lc.info?.(`${name} (${pid}) exited with code (${code})`);\n    }\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    // Exit only if not draining. If a user-facing worker exits unexpectedly\n    // during a drain, log a warning but let other user-facing workers drain.\n    if (log === 'error') {\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n  try {\n    await run();\n    // oxlint-disable-next-line no-console\n    console.info(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    // oxlint-disable-next-line no-console\n    console.error(`pid ${pid} exiting with error`, e);\n    process.exit(-1);\n  }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;AA2BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,UAAU;;;;;AAM5C,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IAAI,kBAAkB,KAAK,kBAAkB,GACvD,CACF;AAKD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,KAAW,GAAG,CAAC;AAGvC,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,SAA+B,WAAW;AACpD,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;EAEnB,IAAI,SAAS;AACb,OAAK,GAAG,UAAU,MAAM,WAAW;AACjC,YAAS;AACT,SAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;IAClD;AAMF,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,SAAS,UAAU,UAC1B,cAAc,KAAK,GAAG,KAAK,OAC3B,IACD,CACF;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;GAIzB,MAAM,MAAM,SAAS,KAAK,MAAA,WAAiB,SAAS,IAAI,SAAS;AACjE,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,QAAQ,SAAS,OAAO,GAAG;;EAG/C,MAAM,MAAM,MAAA,eAAqB,IAAI,UAAU;AAC/C,MAAI,IACF,OAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,IAAI,OAAO,GAAG;WAC1D,SAAS,EAClB,OAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;MAEzE,OAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,GAAG;AAIhE,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAKtB,MAAI,QAAQ,QACV,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;AAIpB,IAAM,2BAA2B;;;;;;;;;;;;AAajC,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts

@@ -13,8 +13,8 @@ import type { WriteWorkerClient } from './write-worker-client.ts';
  */
 export declare class IncrementalSyncer {
     #private;
-    constructor(taskID: string, id: string, changeStreamer: ChangeStreamer, statusDb: Database, worker: WriteWorkerClient, mode: ReplicatorMode, publishReplicationStatus: boolean);
-    run(lc: LogContext): Promise<void>;
+    constructor(lc: LogContext, taskID: string, id: string, changeStreamer: ChangeStreamer, statusDb: Database, worker: WriteWorkerClient, mode: ReplicatorMode, publishReplicationStatus: boolean);
+    run(): Promise<void>;
     subscribe(): Source<ReplicaState>;
     stop(lc: LogContext, err?: unknown): void;
 }

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAE3D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,EAEL,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;AAK/C,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAClE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;gBAmB1B,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,wBAAwB,EAAE,OAAO;IAY7B,GAAG,CAAC,EAAE,EAAE,UAAU;IA0IxB,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAIjC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO;CAGnC"}
+{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAE3D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,EAEL,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;AAK/C,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;gBAqB1B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,wBAAwB,EAAE,OAAO;IAc7B,GAAG;IAsJT,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAIjC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO;CAGnC"}