@rocicorp/zero 0.25.9-canary.2 → 0.25.9

package/out/zero/package.json.js

@@ -1,4 +1,4 @@
-const version = "0.25.9-canary.2";
+const version = "0.25.9";
 const packageJson = {
   version
 };

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -129,7 +129,7 @@ class ChangeStreamerImpl {
           }
           this.#storer.store([watermark, change]);
           this.#forwarder.forward([watermark, change]);
-          if (type === "commit") {
+          if (type === "commit" || type === "rollback") {
             watermark = null;
           }
           const readyForMore = this.#storer.readyForMore();

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.js","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Sink, Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type ChangeStreamMessage,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeSourceUpstream} from '../change-source/protocol/current/upstream.ts';\nimport {publishReplicationError} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  subscriptionState: SubscriptionState,\n  autoReset: boolean,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    autoReset,\n    setTimeoutFn,\n  );\n}\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n */\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\nexport type ChangeStream = {\n  changes: Source<ChangeStreamMessage>;\n\n  /**\n   * A Sink to push the {@link StatusMessage}s that reflect Commits\n   * that have been successfully stored by the {@link Storer}, or\n   * downstream {@link StatusMessage}s henceforth.\n   */\n  acks: Sink<ChangeSourceUpstream>;\n};\n\n/** Encapsulates an upstream-specific implementation of a stream of Changes. */\nexport interface ChangeSource {\n  /**\n   * Starts a stream of changes starting after the specific watermark,\n   * with a corresponding sink for upstream acknowledgements.\n   */\n  startStream(afterWatermark: string): Promise<ChangeStream>;\n}\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    autoReset: boolean,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n    );\n    this.#forwarder = new Forwarder();\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership();\n    // The storer will, in turn, detect changes to ownership and stop\n    // the change-streamer appropriately.\n    this.#storer\n      .run()\n      .then(() => this.stop())\n      .catch(e => this.stop(e));\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      try {\n        const startAfter = await this.#storer.getLastWatermarkToStartStream();\n        const stream = await this.#source.startStream(startAfter);\n        this.#stream = stream;\n        this.#state.resetBackoff();\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              this.#storer.status(change); // storer acks once it gets through its queue\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          this.#storer.store([watermark, change]);\n          this.#forwarder.forward([watermark, change]);\n\n          if (type === 'commit') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, ['rollback', {tag: 'rollback'}]]);\n      }\n\n      await this.#state.backoff(this.#lc, err);\n    }\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n\n      if (initial) {\n        this.scheduleCleanup(watermark);\n      }\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        this.#lc.info?.(`Purged ${deleted} changes before ${earliestInitial}`);\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n"],"names":["ErrorType.WrongReplicaVersion"],"mappings":";;;;;;;;;;;;;;;AA6CA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,mBACA,WACA,eAAe,YACiB;AAEhC,QAAM,yBAAyB,IAAI,UAAU,KAAK;AAClD,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,EAAC,mBAAkB;AACzB,SAAO,IAAI;AAAA,IACT;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AA4KA,MAAM,mBAAoD;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,yCAAyB,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUzB,WAAW,SAAA;AAAA,EAEX,aAAa;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF;AAAA,EAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,WACA,eAAe,YACf;AACA,SAAK,KAAK;AACV,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AACxD,SAAK,SAAS;AACd,SAAK,YAAY;AACjB,SAAK,kBAAkB;AACvB,SAAK,UAAU;AACf,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,CAAA,aAAY,KAAK,SAAS,KAAK,KAAK,CAAC,UAAU,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;AAAA,MACxE,CAAA,QAAO,KAAK,KAAK,GAAG;AAAA,IAAA;AAEtB,SAAK,aAAa,IAAI,UAAA;AACtB,SAAK,aAAa;AAClB,SAAK,SAAS,IAAI,aAAa,KAAK,IAAI,QAAW,YAAY;AAAA,EACjE;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,IAAI,OAAO,wBAAwB;AAIxC,UAAM,KAAK,QAAQ,gBAAA;AAGnB,SAAK,QACF,IAAA,EACA,KAAK,MAAM,KAAK,KAAA,CAAM,EACtB,MAAM,CAAA,MAAK,KAAK,KAAK,CAAC,CAAC;AAE1B,WAAO,KAAK,OAAO,aAAa;AAC9B,UAAI;AACJ,UAAI,YAA2B;AAC/B,UAAI;AACF,cAAM,aAAa,MAAM,KAAK,QAAQ,8BAAA;AACtC,cAAM,SAAS,MAAM,KAAK,QAAQ,YAAY,UAAU;AACxD,aAAK,UAAU;AACf,aAAK,OAAO,aAAA;AACZ,oBAAY;AAEZ,yBAAiB,UAAU,OAAO,SAAS;AACzC,gBAAM,CAAC,MAAM,GAAG,IAAI;AACpB,kBAAQ,MAAA;AAAA,YACN,KAAK;AACH,mBAAK,QAAQ,OAAO,MAAM;AAC1B;AAAA,YACF,KAAK;AACH,oBAAM,KAAK,sBAAsB,GAAG;AACpC;AAAA;AAAA,YACF,KAAK;AACH,0BAAY,OAAO,CAAC,EAAE;AACtB;AAAA,YACF,KAAK;AACH,kBAAI,cAAc,OAAO,CAAC,EAAE,WAAW;AACrC,sBAAM,IAAI;AAAA,kBACR,oBAAoB,OAAO,CAAC,EAAE,SAAS,qCAAqC,SAAS;AAAA,gBAAA;AAAA,cAEzF;AACA,mBAAK,WAAW,IAAI,CAAC;AACrB;AAAA,YACF;AACE,kBAAI,cAAc,MAAM;AACtB,sBAAM,IAAI;AAAA,kBACR,GAAG,IAAI,YAAY,IAAI,GAAG;AAAA,gBAAA;AAAA,cAE9B;AACA;AAAA,UAAA;AAGJ,eAAK,QAAQ,MAAM,CAAC,WAAW,MAAM,CAAC;AACtC,eAAK,WAAW,QAAQ,CAAC,WAAW,MAAM,CAAC;AAE3C,cAAI,SAAS,UAAU;AACrB,wBAAY;AAAA,UACd;AAGA,gBAAM,eAAe,KAAK,QAAQ,aAAA;AAClC,cAAI,cAAc;AAChB,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR,UAAA;AACE,aAAK,SAAS,QAAQ,OAAA;AACtB,aAAK,UAAU;AAAA,MACjB;AAGA,UAAI,WAAW;AACb,aAAK,IAAI,OAAO,oCAAoC,SAAS,EAAE;AAC/D,aAAK,QAAQ,MAAA;AACb,aAAK,WAAW,QAAQ,CAAC,WAAW,CAAC,YAAY,EAAC,KAAK,WAAA,CAAW,CAAC,CAAC;AAAA,MACtE;AAEA,YAAM,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AAAA,IACzC;AACA,SAAK,IAAI,OAAO,wBAAwB;AAAA,EAC1C;AAAA,EAEA,MAAM,sBAAsB,KAA6B;AACvD,SAAK,IAAI,OAAO,4BAA4B,GAAG;AAC/C,UAAM,EAAC,QAAO;AAEd,YAAQ,KAAA;AAAA,MACN,KAAK;AACH,cAAM,kBAAkB,KAAK,WAAW,KAAK,MAAM;AACnD,cAAM;AAAA,UACJ,KAAK;AAAA,UACL;AAAA,UACA,IAAI,WAAW;AAAA,UACf,IAAI;AAAA,QAAA;AAEN,YAAI,KAAK,YAAY;AACnB,eAAK,IAAI,OAAO,8BAA8B;AAC9C,gBAAM,KAAK,KAAK,IAAI,iBAAiB;AAAA,QACvC;AACA;AAAA,MACF;AACE,oBAAe;AAAA,IAAA;AAAA,EAErB;AAAA,EAEA,UAAU,KAAqD;AAC7D,UAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,QAAI,SAAS,WAAW;AACtB,WAAK,SAAS,QAAA;AAAA,IAChB;AACA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM,KAAK,WAAW,OAAO,UAAU;AAAA,IAAA,CACjD;AACD,UAAM,aAAa,IAAI;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,mBAAmB,KAAK,iBAAiB;AAC3C,WAAK,IAAI;AAAA,QACP,2CAA2C,cAAc;AAAA,MAAA;AAE3D,iBAAW;AAAA,QACTA;AAAAA,QACA,8BACE,KAAK,eACP,eAAe,cAAc;AAAA,MAAA;AAAA,IAEjC,OAAO;AACL,WAAK,IAAI,QAAQ,qBAAqB,WAAW,EAAE,EAAE;AAErD,WAAK,WAAW,IAAI,UAAU;AAC9B,WAAK,QAAQ,QAAQ,YAAY,IAAI;AAErC,UAAI,SAAS;AACX,aAAK,gBAAgB,SAAS;AAAA,MAChC;AAAA,IACF;AACA,WAAO,QAAQ,QAAQ,UAAU;AAAA,EACnC;AAAA,EAEA,gBAAgB,WAAmB;AACjC,UAAM,WAAW,KAAK,mBAAmB;AACzC,SAAK,mBAAmB,IAAI,SAAS;AAErC,QAAI,aAAa,GAAG;AAClB,WAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,MAAM,oBAGH;AACD,UAAM,eAAe,MAAM,KAAK,QAAQ,0BAAA;AACxC,WAAO;AAAA,MACL,gBAAgB,KAAK;AAAA,MACrB,cAAc,gBAAgB,KAAK;AAAA,IAAA;AAAA,EAEvC;AAAA,EAEA,MAAM,mBAAkC;AACtC,UAAM,UAAU,CAAC,GAAG,KAAK,kBAAkB;AAC3C,QAAI,QAAQ,WAAW,GAAG;AACxB,WAAK,IAAI,OAAO,4CAA4C;AAC5D;AAAA,IACF;AACA,UAAM,UAAU,CAAC,GAAG,KAAK,WAAW,SAAS;AAC7C,QAAI,QAAQ,WAAW,GAAG;AAGxB,WAAK,IAAI,OAAO,mCAAmC;AACnD;AAAA,IACF;AACA,QAAI;AACF,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,UAAI,kBAAkB,iBAAiB;AACrC,aAAK,IAAI;AAAA,UACP,yCAAyC,eAAe,MAAM,eAAe;AAAA,QAAA;AAAA,MAEjF,OAAO;AACL,cAAM,UAAU,MAAM,KAAK,QAAQ,mBAAmB,eAAe;AACrE,aAAK,IAAI,OAAO,UAAU,OAAO,mBAAmB,eAAe,EAAE;AACrE,aAAK,mBAAmB,OAAO,eAAe;AAAA,MAChD;AAAA,IACF,UAAA;AACE,UAAI,KAAK,mBAAmB,MAAM;AAEhC,aAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,KAAe;AACxB,SAAK,OAAO,KAAK,KAAK,KAAK,GAAG;AAC9B,SAAK,SAAS,QAAQ,OAAA;AACtB,UAAM,KAAK,QAAQ,KAAA;AAAA,EACrB;AACF;AAiBA,MAAM,mBAAmB,6BAA6B;"}
+{"version":3,"file":"change-streamer-service.js","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Sink, Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type ChangeStreamMessage,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeSourceUpstream} from '../change-source/protocol/current/upstream.ts';\nimport {publishReplicationError} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  subscriptionState: SubscriptionState,\n  autoReset: boolean,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    autoReset,\n    setTimeoutFn,\n  );\n}\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n */\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\nexport type ChangeStream = {\n  changes: Source<ChangeStreamMessage>;\n\n  /**\n   * A Sink to push the {@link StatusMessage}s that reflect Commits\n   * that have been successfully stored by the {@link Storer}, or\n   * downstream {@link StatusMessage}s henceforth.\n   */\n  acks: Sink<ChangeSourceUpstream>;\n};\n\n/** Encapsulates an upstream-specific implementation of a stream of Changes. */\nexport interface ChangeSource {\n  /**\n   * Starts a stream of changes starting after the specific watermark,\n   * with a corresponding sink for upstream acknowledgements.\n   */\n  startStream(afterWatermark: string): Promise<ChangeStream>;\n}\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    autoReset: boolean,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n    );\n    this.#forwarder = new Forwarder();\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership();\n    // The storer will, in turn, detect changes to ownership and stop\n    // the change-streamer appropriately.\n    this.#storer\n      .run()\n      .then(() => this.stop())\n      .catch(e => this.stop(e));\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      try {\n        const startAfter = await this.#storer.getLastWatermarkToStartStream();\n        const stream = await this.#source.startStream(startAfter);\n        this.#stream = stream;\n        this.#state.resetBackoff();\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              this.#storer.status(change); // storer acks once it gets through its queue\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          this.#storer.store([watermark, change]);\n          this.#forwarder.forward([watermark, change]);\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await readyForMore;\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, ['rollback', {tag: 'rollback'}]]);\n      }\n\n      await this.#state.backoff(this.#lc, err);\n    }\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n\n      if (initial) {\n        this.scheduleCleanup(watermark);\n      }\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        this.#lc.info?.(`Purged ${deleted} changes before ${earliestInitial}`);\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await this.#storer.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n"],"names":["ErrorType.WrongReplicaVersion"],"mappings":";;;;;;;;;;;;;;;AA6CA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,mBACA,WACA,eAAe,YACiB;AAEhC,QAAM,yBAAyB,IAAI,UAAU,KAAK;AAClD,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,EAAC,mBAAkB;AACzB,SAAO,IAAI;AAAA,IACT;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AA4KA,MAAM,mBAAoD;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,yCAAyB,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUzB,WAAW,SAAA;AAAA,EAEX,aAAa;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF;AAAA,EAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,WACA,eAAe,YACf;AACA,SAAK,KAAK;AACV,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AACxD,SAAK,SAAS;AACd,SAAK,YAAY;AACjB,SAAK,kBAAkB;AACvB,SAAK,UAAU;AACf,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,CAAA,aAAY,KAAK,SAAS,KAAK,KAAK,CAAC,UAAU,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;AAAA,MACxE,CAAA,QAAO,KAAK,KAAK,GAAG;AAAA,IAAA;AAEtB,SAAK,aAAa,IAAI,UAAA;AACtB,SAAK,aAAa;AAClB,SAAK,SAAS,IAAI,aAAa,KAAK,IAAI,QAAW,YAAY;AAAA,EACjE;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,IAAI,OAAO,wBAAwB;AAIxC,UAAM,KAAK,QAAQ,gBAAA;AAGnB,SAAK,QACF,IAAA,EACA,KAAK,MAAM,KAAK,KAAA,CAAM,EACtB,MAAM,CAAA,MAAK,KAAK,KAAK,CAAC,CAAC;AAE1B,WAAO,KAAK,OAAO,aAAa;AAC9B,UAAI;AACJ,UAAI,YAA2B;AAC/B,UAAI;AACF,cAAM,aAAa,MAAM,KAAK,QAAQ,8BAAA;AACtC,cAAM,SAAS,MAAM,KAAK,QAAQ,YAAY,UAAU;AACxD,aAAK,UAAU;AACf,aAAK,OAAO,aAAA;AACZ,oBAAY;AAEZ,yBAAiB,UAAU,OAAO,SAAS;AACzC,gBAAM,CAAC,MAAM,GAAG,IAAI;AACpB,kBAAQ,MAAA;AAAA,YACN,KAAK;AACH,mBAAK,QAAQ,OAAO,MAAM;AAC1B;AAAA,YACF,KAAK;AACH,oBAAM,KAAK,sBAAsB,GAAG;AACpC;AAAA;AAAA,YACF,KAAK;AACH,0BAAY,OAAO,CAAC,EAAE;AACtB;AAAA,YACF,KAAK;AACH,kBAAI,cAAc,OAAO,CAAC,EAAE,WAAW;AACrC,sBAAM,IAAI;AAAA,kBACR,oBAAoB,OAAO,CAAC,EAAE,SAAS,qCAAqC,SAAS;AAAA,gBAAA;AAAA,cAEzF;AACA,mBAAK,WAAW,IAAI,CAAC;AACrB;AAAA,YACF;AACE,kBAAI,cAAc,MAAM;AACtB,sBAAM,IAAI;AAAA,kBACR,GAAG,IAAI,YAAY,IAAI,GAAG;AAAA,gBAAA;AAAA,cAE9B;AACA;AAAA,UAAA;AAGJ,eAAK,QAAQ,MAAM,CAAC,WAAW,MAAM,CAAC;AACtC,eAAK,WAAW,QAAQ,CAAC,WAAW,MAAM,CAAC;AAE3C,cAAI,SAAS,YAAY,SAAS,YAAY;AAC5C,wBAAY;AAAA,UACd;AAGA,gBAAM,eAAe,KAAK,QAAQ,aAAA;AAClC,cAAI,cAAc;AAChB,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR,UAAA;AACE,aAAK,SAAS,QAAQ,OAAA;AACtB,aAAK,UAAU;AAAA,MACjB;AAGA,UAAI,WAAW;AACb,aAAK,IAAI,OAAO,oCAAoC,SAAS,EAAE;AAC/D,aAAK,QAAQ,MAAA;AACb,aAAK,WAAW,QAAQ,CAAC,WAAW,CAAC,YAAY,EAAC,KAAK,WAAA,CAAW,CAAC,CAAC;AAAA,MACtE;AAEA,YAAM,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AAAA,IACzC;AACA,SAAK,IAAI,OAAO,wBAAwB;AAAA,EAC1C;AAAA,EAEA,MAAM,sBAAsB,KAA6B;AACvD,SAAK,IAAI,OAAO,4BAA4B,GAAG;AAC/C,UAAM,EAAC,QAAO;AAEd,YAAQ,KAAA;AAAA,MACN,KAAK;AACH,cAAM,kBAAkB,KAAK,WAAW,KAAK,MAAM;AACnD,cAAM;AAAA,UACJ,KAAK;AAAA,UACL;AAAA,UACA,IAAI,WAAW;AAAA,UACf,IAAI;AAAA,QAAA;AAEN,YAAI,KAAK,YAAY;AACnB,eAAK,IAAI,OAAO,8BAA8B;AAC9C,gBAAM,KAAK,KAAK,IAAI,iBAAiB;AAAA,QACvC;AACA;AAAA,MACF;AACE,oBAAe;AAAA,IAAA;AAAA,EAErB;AAAA,EAEA,UAAU,KAAqD;AAC7D,UAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,QAAI,SAAS,WAAW;AACtB,WAAK,SAAS,QAAA;AAAA,IAChB;AACA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM,KAAK,WAAW,OAAO,UAAU;AAAA,IAAA,CACjD;AACD,UAAM,aAAa,IAAI;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,mBAAmB,KAAK,iBAAiB;AAC3C,WAAK,IAAI;AAAA,QACP,2CAA2C,cAAc;AAAA,MAAA;AAE3D,iBAAW;AAAA,QACTA;AAAAA,QACA,8BACE,KAAK,eACP,eAAe,cAAc;AAAA,MAAA;AAAA,IAEjC,OAAO;AACL,WAAK,IAAI,QAAQ,qBAAqB,WAAW,EAAE,EAAE;AAErD,WAAK,WAAW,IAAI,UAAU;AAC9B,WAAK,QAAQ,QAAQ,YAAY,IAAI;AAErC,UAAI,SAAS;AACX,aAAK,gBAAgB,SAAS;AAAA,MAChC;AAAA,IACF;AACA,WAAO,QAAQ,QAAQ,UAAU;AAAA,EACnC;AAAA,EAEA,gBAAgB,WAAmB;AACjC,UAAM,WAAW,KAAK,mBAAmB;AACzC,SAAK,mBAAmB,IAAI,SAAS;AAErC,QAAI,aAAa,GAAG;AAClB,WAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,MAAM,oBAGH;AACD,UAAM,eAAe,MAAM,KAAK,QAAQ,0BAAA;AACxC,WAAO;AAAA,MACL,gBAAgB,KAAK;AAAA,MACrB,cAAc,gBAAgB,KAAK;AAAA,IAAA;AAAA,EAEvC;AAAA,EAEA,MAAM,mBAAkC;AACtC,UAAM,UAAU,CAAC,GAAG,KAAK,kBAAkB;AAC3C,QAAI,QAAQ,WAAW,GAAG;AACxB,WAAK,IAAI,OAAO,4CAA4C;AAC5D;AAAA,IACF;AACA,UAAM,UAAU,CAAC,GAAG,KAAK,WAAW,SAAS;AAC7C,QAAI,QAAQ,WAAW,GAAG;AAGxB,WAAK,IAAI,OAAO,mCAAmC;AACnD;AAAA,IACF;AACA,QAAI;AACF,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,UAAI,kBAAkB,iBAAiB;AACrC,aAAK,IAAI;AAAA,UACP,yCAAyC,eAAe,MAAM,eAAe;AAAA,QAAA;AAAA,MAEjF,OAAO;AACL,cAAM,UAAU,MAAM,KAAK,QAAQ,mBAAmB,eAAe;AACrE,aAAK,IAAI,OAAO,UAAU,OAAO,mBAAmB,eAAe,EAAE;AACrE,aAAK,mBAAmB,OAAO,eAAe;AAAA,MAChD;AAAA,IACF,UAAA;AACE,UAAI,KAAK,mBAAmB,MAAM;AAEhC,aAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,KAAe;AACxB,SAAK,OAAO,KAAK,KAAK,KAAK,GAAG;AAC9B,SAAK,SAAS,QAAQ,OAAA;AACtB,UAAM,KAAK,QAAQ,KAAA;AAAA,EACrB;AACF;AAiBA,MAAM,mBAAmB,6BAA6B;"}

package/out/zero-cache/src/services/change-streamer/storer.js

@@ -29,7 +29,7 @@ class Storer {
   #queue = new Queue();
   #running = false;
   constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onConsumed, onFatal) {
-    this.#lc = lc;
+    this.#lc = lc.withContext("component", "change-log");
     this.#shard = shard;
     this.#taskID = taskID;
     this.#discoveryAddress = discoveryAddress;

package/out/zero-cache/src/services/change-streamer/storer.js.map

@@ -1 +1 @@
-{"version":3,"file":"storer.js","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport postgres from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {disableStatementTimeout, type PostgresDB} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {type Commit} from '../change-source/protocol/current/downstream.ts';\nimport type {StatusMessage} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type ChangeEntry} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n  AutoResetSignal,\n  markResetRequired,\n  type ReplicationState,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n  subscriber: Subscriber;\n  mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n  | ['change', WatermarkedChange]\n  | ['ready', callback: () => void]\n  | ['subscriber', SubscriberAndMode]\n  | StatusMessage\n  | ['abort']\n  | 'stop';\n\ntype PendingTransaction = {\n  pool: TransactionPool;\n  preCommitWatermark: string;\n  pos: number;\n  startingReplicationState: Promise<ReplicationState>;\n};\n\n// Technically, any threshold is fine because the point of back pressure\n// is to adjust the rate of incoming messages, and the size of the pending\n// work queue does not affect that mechanism.\n//\n// However, it is theoretically possible to exceed the available memory if\n// the size of changes is very large. This threshold can be improved by\n// roughly measuring the size of the enqueued contents and setting the\n// threshold based on available memory.\n//\n// TODO: switch to a message size-based thresholding when migrating over\n// to stringified JSON messages, which will bound the computation involved\n// in measuring the size of row messages.\nconst QUEUE_SIZE_BACK_PRESSURE_THRESHOLD = 100_000;\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n *   `replicaVersion`, which is the version at which initial sync\n *   was performed, and any attempts to catchup from a different\n *   point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n *   a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n *   subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n  readonly id = 'storer';\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #taskID: string;\n  readonly #discoveryAddress: string;\n  readonly #discoveryProtocol: string;\n  readonly #db: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #onConsumed: (c: Commit | StatusMessage) => void;\n  readonly #onFatal: (err: Error) => void;\n  readonly #queue = new Queue<QueueEntry>();\n\n  #running = false;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    db: PostgresDB,\n    replicaVersion: string,\n    onConsumed: (c: Commit | StatusMessage) => void,\n    onFatal: (err: Error) => void,\n  ) {\n    this.#lc = lc;\n    this.#shard = shard;\n    this.#taskID = taskID;\n    this.#discoveryAddress = discoveryAddress;\n    this.#discoveryProtocol = discoveryProtocol;\n    this.#db = db;\n    this.#replicaVersion = replicaVersion;\n    this.#onConsumed = onConsumed;\n    this.#onFatal = onFatal;\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async assumeOwnership() {\n    const db = this.#db;\n    const owner = this.#taskID;\n    const ownerAddress = this.#discoveryAddress;\n    const ownerProtocol = this.#discoveryProtocol;\n    // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n    const addressWithProtocol =\n      ownerProtocol === 'ws'\n        ? ownerAddress\n        : `${ownerProtocol}://${ownerAddress}`;\n    await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n    this.#lc.info?.(`assumed ownership at ${addressWithProtocol}`);\n  }\n\n  async getLastWatermarkToStartStream(): Promise<string> {\n    // Before starting or restarting a stream from the change source,\n    // wait for all queued changes to be processed so that we pick up\n    // from the right spot.\n    const {promise: ready, resolve} = resolver();\n    this.#queue.enqueue(['ready', resolve]);\n    await ready;\n\n    const [{lastWatermark}] = await this.#db<{lastWatermark: string}[]>`\n      SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`;\n    return lastWatermark;\n  }\n\n  async getMinWatermarkForCatchup(): Promise<string | null> {\n    const [{minWatermark}] = await this.#db<\n      {minWatermark: string | null}[]\n    > /*sql*/ `\n      SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n    return minWatermark;\n  }\n\n  purgeRecordsBefore(watermark: string): Promise<number> {\n    return this.#db.begin(Mode.SERIALIZABLE, async sql => {\n      disableStatementTimeout(sql);\n\n      // Check ownership before performing the purge. The server is expected to\n      // exit immediately when an ownership change is detected, but checking\n      // explicitly guards against race conditions.\n      const [{owner}] = await sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')}`;\n      if (owner !== this.#taskID) {\n        this.#lc.warn?.(\n          `Ignoring change log purge request (${watermark}) while not owner`,\n        );\n        return 0;\n      }\n\n      const [{deleted}] = await sql<{deleted: bigint}[]>`\n        WITH purged AS (\n          DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n            RETURNING watermark, pos\n        ) SELECT COUNT(*) as deleted FROM purged;`;\n      return Number(deleted);\n    });\n  }\n\n  store(entry: WatermarkedChange) {\n    this.#queue.enqueue(['change', entry]);\n  }\n\n  abort() {\n    this.#queue.enqueue(['abort']);\n  }\n\n  status(s: StatusMessage) {\n    this.#queue.enqueue(s);\n  }\n\n  catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n    this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n  }\n\n  #readyForMore: Resolver<void> | null = null;\n\n  readyForMore(): Promise<void> | undefined {\n    if (!this.#running) {\n      return undefined;\n    }\n    if (\n      this.#readyForMore === null &&\n      this.#queue.size() > QUEUE_SIZE_BACK_PRESSURE_THRESHOLD\n    ) {\n      this.#lc.warn?.(\n        `applying back pressure with ${this.#queue.size()} queued changes`,\n      );\n      this.#readyForMore = resolver();\n    }\n    return this.#readyForMore?.promise;\n  }\n\n  #maybeReleaseBackPressure() {\n    if (\n      this.#readyForMore !== null &&\n      // Wait for at least 10% of the threshold to free up.\n      this.#queue.size() < QUEUE_SIZE_BACK_PRESSURE_THRESHOLD * 0.9\n    ) {\n      this.#lc.info?.(\n        `releasing back pressure with ${this.#queue.size()} queued changes`,\n      );\n      this.#readyForMore.resolve();\n      this.#readyForMore = null;\n    }\n  }\n\n  async run() {\n    this.#running = true;\n    try {\n      await this.#processQueue();\n    } finally {\n      this.#running = false;\n      this.#lc.info?.('storer stopped');\n    }\n  }\n\n  async #processQueue() {\n    let tx: PendingTransaction | null = null;\n    let msg: QueueEntry | false;\n\n    const catchupQueue: SubscriberAndMode[] = [];\n    while ((msg = await this.#queue.dequeue()) !== 'stop') {\n      this.#maybeReleaseBackPressure();\n\n      const [msgType] = msg;\n      switch (msgType) {\n        case 'ready': {\n          const signalReady = msg[1];\n          signalReady();\n          continue;\n        }\n        case 'subscriber': {\n          const subscriber = msg[1];\n          if (tx) {\n            catchupQueue.push(subscriber); // Wait for the current tx to complete.\n          } else {\n            await this.#startCatchup([subscriber]); // Catch up immediately.\n          }\n          continue;\n        }\n        case 'status':\n          this.#onConsumed(msg);\n          continue;\n        case 'abort': {\n          if (tx) {\n            tx.pool.abort();\n            await tx.pool.done();\n            tx = null;\n          }\n          continue;\n        }\n      }\n      // msgType === 'change'\n      const [watermark, downstream] = msg[1];\n      const [tag, change] = downstream;\n      if (tag === 'begin') {\n        assert(!tx, 'received BEGIN in the middle of a transaction');\n        const {promise, resolve, reject} = resolver<ReplicationState>();\n        tx = {\n          pool: new TransactionPool(\n            this.#lc.withContext('watermark', watermark),\n            Mode.SERIALIZABLE,\n          ),\n          preCommitWatermark: watermark,\n          pos: 0,\n          startingReplicationState: promise,\n        };\n        tx.pool.run(this.#db);\n        // Pipeline a read of the current ReplicationState,\n        // which will be checked before committing.\n        void tx.pool.process(tx => {\n          tx<ReplicationState[]>`\n          SELECT * FROM ${this.#cdc('replicationState')}`.then(\n            ([result]) => resolve(result),\n            reject,\n          );\n          return [];\n        });\n      } else {\n        assert(tx, `received ${tag} outside of transaction`);\n        tx.pos++;\n      }\n\n      const entry = {\n        watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n        precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n        pos: tx.pos,\n        change: change as unknown as JSONValue,\n      };\n\n      const processed = tx.pool.process(tx => [\n        tx`\n        INSERT INTO ${this.#cdc('changeLog')} ${tx(entry)}`,\n      ]);\n\n      if (tag === 'data' && tx.pos % 10_000 === 0) {\n        // Backpressure is exerted on commit when awaiting tx.pool.done().\n        // However, backpressure checks need to be regularly done for\n        // very large transactions in order to avoid memory blowup.\n        await processed;\n      }\n\n      if (tag === 'commit') {\n        const {owner} = await tx.startingReplicationState;\n        if (owner !== this.#taskID) {\n          // Ownership change reflected in the replicationState read in 'begin'.\n          tx.pool.fail(\n            new AbortError(`changeLog ownership has been assumed by ${owner}`),\n          );\n        } else {\n          // Update the replication state.\n          const lastWatermark = watermark;\n          void tx.pool.process(tx => [\n            tx`\n            UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n          ]);\n          tx.pool.setDone();\n        }\n\n        try {\n          await tx.pool.done();\n        } catch (e) {\n          if (\n            e instanceof postgres.PostgresError &&\n            e.code === PG_SERIALIZATION_FAILURE\n          ) {\n            // Ownership change happened after the replicationState was read in 'begin'.\n            throw new AbortError(`changeLog ownership has changed`, {cause: e});\n          }\n          throw e;\n        }\n\n        tx = null;\n\n        // ACK the LSN to the upstream Postgres.\n        this.#onConsumed(downstream);\n\n        // Before beginning the next transaction, open a READONLY snapshot to\n        // concurrently catchup any queued subscribers.\n        await this.#startCatchup(catchupQueue.splice(0));\n      } else if (tag === 'rollback') {\n        // Aborted transactions are not stored in the changeLog. Abort the current tx\n        // and process catchup of subscribers that were waiting for it to end.\n        tx.pool.abort();\n        await tx.pool.done();\n        tx = null;\n\n        await this.#startCatchup(catchupQueue.splice(0));\n      }\n    }\n  }\n\n  async #startCatchup(subs: SubscriberAndMode[]) {\n    if (subs.length === 0) {\n      return;\n    }\n\n    const reader = new TransactionPool(\n      this.#lc.withContext('pool', 'catchup'),\n      Mode.READONLY,\n    );\n    reader.run(this.#db);\n\n    // Ensure that the transaction has started (and is thus holding a snapshot\n    // of the database) before continuing on to commit more changes. This is\n    // done by waiting for a no-op task to be processed by the pool, which\n    // indicates that the BEGIN statement has been sent to the database.\n    await reader.processReadTask(() => {});\n\n    // Run the actual catchup queries in the background. Errors are handled in\n    // #catchup() by disconnecting the associated subscriber.\n    void Promise.all(subs.map(sub => this.#catchup(sub, reader))).finally(() =>\n      reader.setDone(),\n    );\n  }\n\n  async #catchup(\n    {subscriber: sub, mode}: SubscriberAndMode,\n    reader: TransactionPool,\n  ) {\n    try {\n      await reader.processReadTask(async tx => {\n        const start = Date.now();\n\n        // When starting from initial-sync, there won't be a change with a watermark\n        // equal to the replica version. This is the empty changeLog scenario.\n        let watermarkFound = sub.watermark === this.#replicaVersion;\n        let count = 0;\n        let lastBatchConsumed: Promise<unknown> | undefined;\n\n        for await (const entries of tx<ChangeEntry[]>`\n          SELECT watermark, change FROM ${this.#cdc('changeLog')}\n           WHERE watermark >= ${sub.watermark}\n           ORDER BY watermark, pos`.cursor(2000)) {\n          // Wait for the last batch of entries to be consumed by the\n          // subscriber before sending down the current batch. This pipelining\n          // allows one batch of changes to be received from the change-db\n          // while the previous batch of changes are sent to the subscriber,\n          // resulting in flow control that caps the number of changes\n          // referenced in memory to 2 * batch-size.\n          const start = performance.now();\n          await lastBatchConsumed;\n          const elapsed = performance.now() - start;\n          if (lastBatchConsumed) {\n            (elapsed > 100 ? this.#lc.info : this.#lc.debug)?.(\n              `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n            );\n          }\n\n          for (const entry of entries) {\n            if (entry.watermark === sub.watermark) {\n              // This should be the first entry.\n              // Catchup starts from *after* the watermark.\n              watermarkFound = true;\n            } else if (watermarkFound) {\n              lastBatchConsumed = sub.catchup(toDownstream(entry)).result;\n              count++;\n            } else if (mode === 'backup') {\n              throw new AutoResetSignal(\n                `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n              );\n            } else {\n              this.#lc.warn?.(\n                `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n              );\n              sub.close(\n                ErrorType.WatermarkTooOld,\n                `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n              );\n              return;\n            }\n          }\n        }\n        if (watermarkFound) {\n          await lastBatchConsumed;\n          this.#lc.info?.(\n            `caught up ${sub.id} with ${count} changes (${\n              Date.now() - start\n            } ms)`,\n          );\n        } else {\n          this.#lc.warn?.(\n            `subscriber at watermark ${sub.watermark} is ahead of latest watermark`,\n          );\n        }\n        // Flushes the backlog of messages buffered during catchup and\n        // allows the subscription to forward subsequent messages immediately.\n        sub.setCaughtUp();\n      });\n    } catch (err) {\n      this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n      if (err instanceof AutoResetSignal) {\n        await markResetRequired(this.#db, this.#shard);\n        this.#onFatal(err);\n      }\n      sub.fail(err);\n    }\n  }\n\n  stop() {\n    this.#queue.enqueue('stop');\n    return promiseVoid;\n  }\n}\n\nfunction toDownstream(entry: ChangeEntry): WatermarkedChange {\n  const {watermark, change} = entry;\n  switch (change.tag) {\n    case 'begin':\n      return [watermark, ['begin', change, {commitWatermark: watermark}]];\n    case 'commit':\n      return [watermark, ['commit', change, {watermark}]];\n    case 'rollback':\n      return [watermark, ['rollback', change]];\n    default:\n      return [watermark, ['data', change]];\n  }\n}\n"],"names":["Mode.SERIALIZABLE","tx","Mode.READONLY","start","ErrorType.WatermarkTooOld"],"mappings":";;;;;;;;;;;;;;;;AA2DA,MAAM,qCAAqC;AAgCpC,MAAM,OAA0B;AAAA,EAC5B,KAAK;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,MAAA;AAAA,EAEtB,WAAW;AAAA,EAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA;AACA,SAAK,MAAM;AACX,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,oBAAoB;AACzB,SAAK,qBAAqB;AAC1B,SAAK,MAAM;AACX,SAAK,kBAAkB;AACvB,SAAK,cAAc;AACnB,SAAK,WAAW;AAAA,EAClB;AAAA;AAAA,EAGA,KAAK,OAAe;AAClB,WAAO,KAAK,IAAI,GAAG,UAAU,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;AAAA,EACtD;AAAA,EAEA,MAAM,kBAAkB;AACtB,UAAM,KAAK,KAAK;AAChB,UAAM,QAAQ,KAAK;AACnB,UAAM,eAAe,KAAK;AAC1B,UAAM,gBAAgB,KAAK;AAE3B,UAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,aAAa,MAAM,YAAY;AACxC,UAAM,YAAY,KAAK,KAAK,kBAAkB,CAAC,QAAQ,GAAG,EAAC,OAAO,cAAc,oBAAA,CAAoB,CAAC;AACrG,SAAK,IAAI,OAAO,wBAAwB,mBAAmB,EAAE;AAAA,EAC/D;AAAA,EAEA,MAAM,gCAAiD;AAIrD,UAAM,EAAC,SAAS,OAAO,QAAA,IAAW,SAAA;AAClC,SAAK,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;AACtC,UAAM;AAEN,UAAM,CAAC,EAAC,cAAA,CAAc,IAAI,MAAM,KAAK;AAAA,oCACL,KAAK,KAAK,kBAAkB,CAAC;AAC7D,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,4BAAoD;AACxD,UAAM,CAAC,EAAC,aAAA,CAAa,IAAI,MAAM,KAAK;AAAA,qDAGa,KAAK,KAAK,WAAW,CAAC;AACvE,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,WAAoC;AACrD,WAAO,KAAK,IAAI,MAAMA,cAAmB,OAAM,QAAO;AACpD,8BAAwB,GAAG;AAK3B,YAAM,CAAC,EAAC,OAAM,IAAI,MAAM;AAAA,wBACN,KAAK,KAAK,kBAAkB,CAAC;AAC/C,UAAI,UAAU,KAAK,SAAS;AAC1B,aAAK,IAAI;AAAA,UACP,sCAAsC,SAAS;AAAA,QAAA;AAEjD,eAAO;AAAA,MACT;AAEA,YAAM,CAAC,EAAC,SAAQ,IAAI,MAAM;AAAA;AAAA,wBAER,KAAK,KAAK,WAAW,CAAC,sBAAsB,SAAS;AAAA;AAAA;AAGvE,aAAO,OAAO,OAAO;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,OAA0B;AAC9B,SAAK,OAAO,QAAQ,CAAC,UAAU,KAAK,CAAC;AAAA,EACvC;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,QAAQ,CAAC,OAAO,CAAC;AAAA,EAC/B;AAAA,EAEA,OAAO,GAAkB;AACvB,SAAK,OAAO,QAAQ,CAAC;AAAA,EACvB;AAAA,EAEA,QAAQ,YAAwB,MAAsB;AACpD,SAAK,OAAO,QAAQ,CAAC,cAAc,EAAC,YAAY,KAAA,CAAK,CAAC;AAAA,EACxD;AAAA,EAEA,gBAAuC;AAAA,EAEvC,eAA0C;AACxC,QAAI,CAAC,KAAK,UAAU;AAClB,aAAO;AAAA,IACT;AACA,QACE,KAAK,kBAAkB,QACvB,KAAK,OAAO,KAAA,IAAS,oCACrB;AACA,WAAK,IAAI;AAAA,QACP,+BAA+B,KAAK,OAAO,KAAA,CAAM;AAAA,MAAA;AAEnD,WAAK,gBAAgB,SAAA;AAAA,IACvB;AACA,WAAO,KAAK,eAAe;AAAA,EAC7B;AAAA,EAEA,4BAA4B;AAC1B,QACE,KAAK,kBAAkB;AAAA,IAEvB,KAAK,OAAO,SAAS,qCAAqC,KAC1D;AACA,WAAK,IAAI;AAAA,QACP,gCAAgC,KAAK,OAAO,KAAA,CAAM;AAAA,MAAA;AAEpD,WAAK,cAAc,QAAA;AACnB,WAAK,gBAAgB;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,WAAW;AAChB,QAAI;AACF,YAAM,KAAK,cAAA;AAAA,IACb,UAAA;AACE,WAAK,WAAW;AAChB,WAAK,IAAI,OAAO,gBAAgB;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,MAAM,gBAAgB;AACpB,QAAI,KAAgC;AACpC,QAAI;AAEJ,UAAM,eAAoC,CAAA;AAC1C,YAAQ,MAAM,MAAM,KAAK,OAAO,QAAA,OAAe,QAAQ;AACrD,WAAK,0BAAA;AAEL,YAAM,CAAC,OAAO,IAAI;AAClB,cAAQ,SAAA;AAAA,QACN,KAAK,SAAS;AACZ,gBAAM,cAAc,IAAI,CAAC;AACzB,sBAAA;AACA;AAAA,QACF;AAAA,QACA,KAAK,cAAc;AACjB,gBAAM,aAAa,IAAI,CAAC;AACxB,cAAI,IAAI;AACN,yBAAa,KAAK,UAAU;AAAA,UAC9B,OAAO;AACL,kBAAM,KAAK,cAAc,CAAC,UAAU,CAAC;AAAA,UACvC;AACA;AAAA,QACF;AAAA,QACA,KAAK;AACH,eAAK,YAAY,GAAG;AACpB;AAAA,QACF,KAAK,SAAS;AACZ,cAAI,IAAI;AACN,eAAG,KAAK,MAAA;AACR,kBAAM,GAAG,KAAK,KAAA;AACd,iBAAK;AAAA,UACP;AACA;AAAA,QACF;AAAA,MAAA;AAGF,YAAM,CAAC,WAAW,UAAU,IAAI,IAAI,CAAC;AACrC,YAAM,CAAC,KAAK,MAAM,IAAI;AACtB,UAAI,QAAQ,SAAS;AACnB,eAAO,CAAC,IAAI,+CAA+C;AAC3D,cAAM,EAAC,SAAS,SAAS,OAAA,IAAU,SAAA;AACnC,aAAK;AAAA,UACH,MAAM,IAAI;AAAA,YACR,KAAK,IAAI,YAAY,aAAa,SAAS;AAAA,YAC3CA;AAAAA,UAAK;AAAA,UAEP,oBAAoB;AAAA,UACpB,KAAK;AAAA,UACL,0BAA0B;AAAA,QAAA;AAE5B,WAAG,KAAK,IAAI,KAAK,GAAG;AAGpB,aAAK,GAAG,KAAK,QAAQ,CAAAC,QAAM;AACzBA;AAAAA,0BACgB,KAAK,KAAK,kBAAkB,CAAC,GAAG;AAAA,YAC9C,CAAC,CAAC,MAAM,MAAM,QAAQ,MAAM;AAAA,YAC5B;AAAA,UAAA;AAEF,iBAAO,CAAA;AAAA,QACT,CAAC;AAAA,MACH,OAAO;AACL,eAAO,IAAI,YAAY,GAAG,yBAAyB;AACnD,WAAG;AAAA,MACL;AAEA,YAAM,QAAQ;AAAA,QACZ,WAAW,QAAQ,WAAW,YAAY,GAAG;AAAA,QAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;AAAA,QACtD,KAAK,GAAG;AAAA,QACR;AAAA,MAAA;AAGF,YAAM,YAAY,GAAG,KAAK,QAAQ,CAAAA,QAAM;AAAA,QACtCA;AAAAA,sBACc,KAAK,KAAK,WAAW,CAAC,IAAIA,IAAG,KAAK,CAAC;AAAA,MAAA,CAClD;AAED,UAAI,QAAQ,UAAU,GAAG,MAAM,QAAW,GAAG;AAI3C,cAAM;AAAA,MACR;AAEA,UAAI,QAAQ,UAAU;AACpB,cAAM,EAAC,MAAA,IAAS,MAAM,GAAG;AACzB,YAAI,UAAU,KAAK,SAAS;AAE1B,aAAG,KAAK;AAAA,YACN,IAAI,WAAW,2CAA2C,KAAK,EAAE;AAAA,UAAA;AAAA,QAErE,OAAO;AAEL,gBAAM,gBAAgB;AACtB,eAAK,GAAG,KAAK,QAAQ,CAAAA,QAAM;AAAA,YACzBA;AAAAA,qBACS,KAAK,KAAK,kBAAkB,CAAC,QAAQA,IAAG,EAAC,cAAA,CAAc,CAAC;AAAA,UAAA,CAClE;AACD,aAAG,KAAK,QAAA;AAAA,QACV;AAEA,YAAI;AACF,gBAAM,GAAG,KAAK,KAAA;AAAA,QAChB,SAAS,GAAG;AACV,cACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;AAEA,kBAAM,IAAI,WAAW,mCAAmC,EAAC,OAAO,GAAE;AAAA,UACpE;AACA,gBAAM;AAAA,QACR;AAEA,aAAK;AAGL,aAAK,YAAY,UAAU;AAI3B,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD,WAAW,QAAQ,YAAY;AAG7B,WAAG,KAAK,MAAA;AACR,cAAM,GAAG,KAAK,KAAA;AACd,aAAK;AAEL,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,MAA2B;AAC7C,QAAI,KAAK,WAAW,GAAG;AACrB;AAAA,IACF;AAEA,UAAM,SAAS,IAAI;AAAA,MACjB,KAAK,IAAI,YAAY,QAAQ,SAAS;AAAA,MACtCC;AAAAA,IAAK;AAEP,WAAO,IAAI,KAAK,GAAG;AAMnB,UAAM,OAAO,gBAAgB,MAAM;AAAA,IAAC,CAAC;AAIrC,SAAK,QAAQ,IAAI,KAAK,IAAI,CAAA,QAAO,KAAK,SAAS,KAAK,MAAM,CAAC,CAAC,EAAE;AAAA,MAAQ,MACpE,OAAO,QAAA;AAAA,IAAQ;AAAA,EAEnB;AAAA,EAEA,MAAM,SACJ,EAAC,YAAY,KAAK,KAAA,GAClB,QACA;AACA,QAAI;AACF,YAAM,OAAO,gBAAgB,OAAM,OAAM;AACvC,cAAM,QAAQ,KAAK,IAAA;AAInB,YAAI,iBAAiB,IAAI,cAAc,KAAK;AAC5C,YAAI,QAAQ;AACZ,YAAI;AAEJ,yBAAiB,WAAW;AAAA,0CACM,KAAK,KAAK,WAAW,CAAC;AAAA,gCAChC,IAAI,SAAS;AAAA,oCACT,OAAO,GAAI,GAAG;AAOxC,gBAAMC,SAAQ,YAAY,IAAA;AAC1B,gBAAM;AACN,gBAAM,UAAU,YAAY,IAAA,IAAQA;AACpC,cAAI,mBAAmB;AACrB,aAAC,UAAU,MAAM,KAAK,IAAI,OAAO,KAAK,IAAI;AAAA,cACxC,UAAU,QAAQ,QAAQ,CAAC,CAAC,WAAW,IAAI,EAAE;AAAA,YAAA;AAAA,UAEjD;AAEA,qBAAW,SAAS,SAAS;AAC3B,gBAAI,MAAM,cAAc,IAAI,WAAW;AAGrC,+BAAiB;AAAA,YACnB,WAAW,gBAAgB;AACzB,kCAAoB,IAAI,QAAQ,aAAa,KAAK,CAAC,EAAE;AACrD;AAAA,YACF,WAAW,SAAS,UAAU;AAC5B,oBAAM,IAAI;AAAA,gBACR,+BAA+B,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAAA,YAExF,OAAO;AACL,mBAAK,IAAI;AAAA,gBACP,qCAAqC,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAE5F,kBAAI;AAAA,gBACFC;AAAAA,gBACA,mCAAmC,MAAM,SAAS,eAAe,IAAI,SAAS;AAAA,cAAA;AAEhF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA,YAAI,gBAAgB;AAClB,gBAAM;AACN,eAAK,IAAI;AAAA,YACP,aAAa,IAAI,EAAE,SAAS,KAAK,aAC/B,KAAK,QAAQ,KACf;AAAA,UAAA;AAAA,QAEJ,OAAO;AACL,eAAK,IAAI;AAAA,YACP,2BAA2B,IAAI,SAAS;AAAA,UAAA;AAAA,QAE5C;AAGA,YAAI,YAAA;AAAA,MACN,CAAC;AAAA,IACH,SAAS,KAAK;AACZ,WAAK,IAAI,QAAQ,sCAAsC,IAAI,EAAE,IAAI,GAAG;AACpE,UAAI,eAAe,iBAAiB;AAClC,cAAM,kBAAkB,KAAK,KAAK,KAAK,MAAM;AAC7C,aAAK,SAAS,GAAG;AAAA,MACnB;AACA,UAAI,KAAK,GAAG;AAAA,IACd;AAAA,EACF;AAAA,EAEA,OAAO;AACL,SAAK,OAAO,QAAQ,MAAM;AAC1B,WAAO;AAAA,EACT;AACF;AAEA,SAAS,aAAa,OAAuC;AAC3D,QAAM,EAAC,WAAW,OAAA,IAAU;AAC5B,UAAQ,OAAO,KAAA;AAAA,IACb,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,SAAS,QAAQ,EAAC,iBAAiB,UAAA,CAAU,CAAC;AAAA,IACpE,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,UAAU,QAAQ,EAAC,UAAA,CAAU,CAAC;AAAA,IACpD,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,YAAY,MAAM,CAAC;AAAA,IACzC;AACE,aAAO,CAAC,WAAW,CAAC,QAAQ,MAAM,CAAC;AAAA,EAAA;AAEzC;"}
+{"version":3,"file":"storer.js","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport postgres from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {type JSONValue} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {disableStatementTimeout, type PostgresDB} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {type Commit} from '../change-source/protocol/current/downstream.ts';\nimport type {StatusMessage} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type ChangeEntry} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n  AutoResetSignal,\n  markResetRequired,\n  type ReplicationState,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n  subscriber: Subscriber;\n  mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n  | ['change', WatermarkedChange]\n  | ['ready', callback: () => void]\n  | ['subscriber', SubscriberAndMode]\n  | StatusMessage\n  | ['abort']\n  | 'stop';\n\ntype PendingTransaction = {\n  pool: TransactionPool;\n  preCommitWatermark: string;\n  pos: number;\n  startingReplicationState: Promise<ReplicationState>;\n};\n\n// Technically, any threshold is fine because the point of back pressure\n// is to adjust the rate of incoming messages, and the size of the pending\n// work queue does not affect that mechanism.\n//\n// However, it is theoretically possible to exceed the available memory if\n// the size of changes is very large. This threshold can be improved by\n// roughly measuring the size of the enqueued contents and setting the\n// threshold based on available memory.\n//\n// TODO: switch to a message size-based thresholding when migrating over\n// to stringified JSON messages, which will bound the computation involved\n// in measuring the size of row messages.\nconst QUEUE_SIZE_BACK_PRESSURE_THRESHOLD = 100_000;\n\n/**\n * Handles the storage of changes and the catchup of subscribers\n * that are behind.\n *\n * In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n *\n * **Maintained invariant**: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n *   `replicaVersion`, which is the version at which initial sync\n *   was performed, and any attempts to catchup from a different\n *   point fail.\n *\n * Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must *not* be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n *   a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n *   subscriber such that there is a gap in its replication history.\n *\n * Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n */\nexport class Storer implements Service {\n  readonly id = 'storer';\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #taskID: string;\n  readonly #discoveryAddress: string;\n  readonly #discoveryProtocol: string;\n  readonly #db: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #onConsumed: (c: Commit | StatusMessage) => void;\n  readonly #onFatal: (err: Error) => void;\n  readonly #queue = new Queue<QueueEntry>();\n\n  #running = false;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    db: PostgresDB,\n    replicaVersion: string,\n    onConsumed: (c: Commit | StatusMessage) => void,\n    onFatal: (err: Error) => void,\n  ) {\n    this.#lc = lc.withContext('component', 'change-log');\n    this.#shard = shard;\n    this.#taskID = taskID;\n    this.#discoveryAddress = discoveryAddress;\n    this.#discoveryProtocol = discoveryProtocol;\n    this.#db = db;\n    this.#replicaVersion = replicaVersion;\n    this.#onConsumed = onConsumed;\n    this.#onFatal = onFatal;\n  }\n\n  // For readability in SQL statements.\n  #cdc(table: string) {\n    return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n  }\n\n  async assumeOwnership() {\n    const db = this.#db;\n    const owner = this.#taskID;\n    const ownerAddress = this.#discoveryAddress;\n    const ownerProtocol = this.#discoveryProtocol;\n    // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n    const addressWithProtocol =\n      ownerProtocol === 'ws'\n        ? ownerAddress\n        : `${ownerProtocol}://${ownerAddress}`;\n    await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n    this.#lc.info?.(`assumed ownership at ${addressWithProtocol}`);\n  }\n\n  async getLastWatermarkToStartStream(): Promise<string> {\n    // Before starting or restarting a stream from the change source,\n    // wait for all queued changes to be processed so that we pick up\n    // from the right spot.\n    const {promise: ready, resolve} = resolver();\n    this.#queue.enqueue(['ready', resolve]);\n    await ready;\n\n    const [{lastWatermark}] = await this.#db<{lastWatermark: string}[]>`\n      SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`;\n    return lastWatermark;\n  }\n\n  async getMinWatermarkForCatchup(): Promise<string | null> {\n    const [{minWatermark}] = await this.#db<\n      {minWatermark: string | null}[]\n    > /*sql*/ `\n      SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n    return minWatermark;\n  }\n\n  purgeRecordsBefore(watermark: string): Promise<number> {\n    return this.#db.begin(Mode.SERIALIZABLE, async sql => {\n      disableStatementTimeout(sql);\n\n      // Check ownership before performing the purge. The server is expected to\n      // exit immediately when an ownership change is detected, but checking\n      // explicitly guards against race conditions.\n      const [{owner}] = await sql<ReplicationState[]>`\n        SELECT * FROM ${this.#cdc('replicationState')}`;\n      if (owner !== this.#taskID) {\n        this.#lc.warn?.(\n          `Ignoring change log purge request (${watermark}) while not owner`,\n        );\n        return 0;\n      }\n\n      const [{deleted}] = await sql<{deleted: bigint}[]>`\n        WITH purged AS (\n          DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n            RETURNING watermark, pos\n        ) SELECT COUNT(*) as deleted FROM purged;`;\n      return Number(deleted);\n    });\n  }\n\n  store(entry: WatermarkedChange) {\n    this.#queue.enqueue(['change', entry]);\n  }\n\n  abort() {\n    this.#queue.enqueue(['abort']);\n  }\n\n  status(s: StatusMessage) {\n    this.#queue.enqueue(s);\n  }\n\n  catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n    this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n  }\n\n  #readyForMore: Resolver<void> | null = null;\n\n  readyForMore(): Promise<void> | undefined {\n    if (!this.#running) {\n      return undefined;\n    }\n    if (\n      this.#readyForMore === null &&\n      this.#queue.size() > QUEUE_SIZE_BACK_PRESSURE_THRESHOLD\n    ) {\n      this.#lc.warn?.(\n        `applying back pressure with ${this.#queue.size()} queued changes`,\n      );\n      this.#readyForMore = resolver();\n    }\n    return this.#readyForMore?.promise;\n  }\n\n  #maybeReleaseBackPressure() {\n    if (\n      this.#readyForMore !== null &&\n      // Wait for at least 10% of the threshold to free up.\n      this.#queue.size() < QUEUE_SIZE_BACK_PRESSURE_THRESHOLD * 0.9\n    ) {\n      this.#lc.info?.(\n        `releasing back pressure with ${this.#queue.size()} queued changes`,\n      );\n      this.#readyForMore.resolve();\n      this.#readyForMore = null;\n    }\n  }\n\n  async run() {\n    this.#running = true;\n    try {\n      await this.#processQueue();\n    } finally {\n      this.#running = false;\n      this.#lc.info?.('storer stopped');\n    }\n  }\n\n  async #processQueue() {\n    let tx: PendingTransaction | null = null;\n    let msg: QueueEntry | false;\n\n    const catchupQueue: SubscriberAndMode[] = [];\n    while ((msg = await this.#queue.dequeue()) !== 'stop') {\n      this.#maybeReleaseBackPressure();\n\n      const [msgType] = msg;\n      switch (msgType) {\n        case 'ready': {\n          const signalReady = msg[1];\n          signalReady();\n          continue;\n        }\n        case 'subscriber': {\n          const subscriber = msg[1];\n          if (tx) {\n            catchupQueue.push(subscriber); // Wait for the current tx to complete.\n          } else {\n            await this.#startCatchup([subscriber]); // Catch up immediately.\n          }\n          continue;\n        }\n        case 'status':\n          this.#onConsumed(msg);\n          continue;\n        case 'abort': {\n          if (tx) {\n            tx.pool.abort();\n            await tx.pool.done();\n            tx = null;\n          }\n          continue;\n        }\n      }\n      // msgType === 'change'\n      const [watermark, downstream] = msg[1];\n      const [tag, change] = downstream;\n      if (tag === 'begin') {\n        assert(!tx, 'received BEGIN in the middle of a transaction');\n        const {promise, resolve, reject} = resolver<ReplicationState>();\n        tx = {\n          pool: new TransactionPool(\n            this.#lc.withContext('watermark', watermark),\n            Mode.SERIALIZABLE,\n          ),\n          preCommitWatermark: watermark,\n          pos: 0,\n          startingReplicationState: promise,\n        };\n        tx.pool.run(this.#db);\n        // Pipeline a read of the current ReplicationState,\n        // which will be checked before committing.\n        void tx.pool.process(tx => {\n          tx<ReplicationState[]>`\n          SELECT * FROM ${this.#cdc('replicationState')}`.then(\n            ([result]) => resolve(result),\n            reject,\n          );\n          return [];\n        });\n      } else {\n        assert(tx, `received ${tag} outside of transaction`);\n        tx.pos++;\n      }\n\n      const entry = {\n        watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n        precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n        pos: tx.pos,\n        change: change as unknown as JSONValue,\n      };\n\n      const processed = tx.pool.process(tx => [\n        tx`\n        INSERT INTO ${this.#cdc('changeLog')} ${tx(entry)}`,\n      ]);\n\n      if (tag === 'data' && tx.pos % 10_000 === 0) {\n        // Backpressure is exerted on commit when awaiting tx.pool.done().\n        // However, backpressure checks need to be regularly done for\n        // very large transactions in order to avoid memory blowup.\n        await processed;\n      }\n\n      if (tag === 'commit') {\n        const {owner} = await tx.startingReplicationState;\n        if (owner !== this.#taskID) {\n          // Ownership change reflected in the replicationState read in 'begin'.\n          tx.pool.fail(\n            new AbortError(`changeLog ownership has been assumed by ${owner}`),\n          );\n        } else {\n          // Update the replication state.\n          const lastWatermark = watermark;\n          void tx.pool.process(tx => [\n            tx`\n            UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n          ]);\n          tx.pool.setDone();\n        }\n\n        try {\n          await tx.pool.done();\n        } catch (e) {\n          if (\n            e instanceof postgres.PostgresError &&\n            e.code === PG_SERIALIZATION_FAILURE\n          ) {\n            // Ownership change happened after the replicationState was read in 'begin'.\n            throw new AbortError(`changeLog ownership has changed`, {cause: e});\n          }\n          throw e;\n        }\n\n        tx = null;\n\n        // ACK the LSN to the upstream Postgres.\n        this.#onConsumed(downstream);\n\n        // Before beginning the next transaction, open a READONLY snapshot to\n        // concurrently catchup any queued subscribers.\n        await this.#startCatchup(catchupQueue.splice(0));\n      } else if (tag === 'rollback') {\n        // Aborted transactions are not stored in the changeLog. Abort the current tx\n        // and process catchup of subscribers that were waiting for it to end.\n        tx.pool.abort();\n        await tx.pool.done();\n        tx = null;\n\n        await this.#startCatchup(catchupQueue.splice(0));\n      }\n    }\n  }\n\n  async #startCatchup(subs: SubscriberAndMode[]) {\n    if (subs.length === 0) {\n      return;\n    }\n\n    const reader = new TransactionPool(\n      this.#lc.withContext('pool', 'catchup'),\n      Mode.READONLY,\n    );\n    reader.run(this.#db);\n\n    // Ensure that the transaction has started (and is thus holding a snapshot\n    // of the database) before continuing on to commit more changes. This is\n    // done by waiting for a no-op task to be processed by the pool, which\n    // indicates that the BEGIN statement has been sent to the database.\n    await reader.processReadTask(() => {});\n\n    // Run the actual catchup queries in the background. Errors are handled in\n    // #catchup() by disconnecting the associated subscriber.\n    void Promise.all(subs.map(sub => this.#catchup(sub, reader))).finally(() =>\n      reader.setDone(),\n    );\n  }\n\n  async #catchup(\n    {subscriber: sub, mode}: SubscriberAndMode,\n    reader: TransactionPool,\n  ) {\n    try {\n      await reader.processReadTask(async tx => {\n        const start = Date.now();\n\n        // When starting from initial-sync, there won't be a change with a watermark\n        // equal to the replica version. This is the empty changeLog scenario.\n        let watermarkFound = sub.watermark === this.#replicaVersion;\n        let count = 0;\n        let lastBatchConsumed: Promise<unknown> | undefined;\n\n        for await (const entries of tx<ChangeEntry[]>`\n          SELECT watermark, change FROM ${this.#cdc('changeLog')}\n           WHERE watermark >= ${sub.watermark}\n           ORDER BY watermark, pos`.cursor(2000)) {\n          // Wait for the last batch of entries to be consumed by the\n          // subscriber before sending down the current batch. This pipelining\n          // allows one batch of changes to be received from the change-db\n          // while the previous batch of changes are sent to the subscriber,\n          // resulting in flow control that caps the number of changes\n          // referenced in memory to 2 * batch-size.\n          const start = performance.now();\n          await lastBatchConsumed;\n          const elapsed = performance.now() - start;\n          if (lastBatchConsumed) {\n            (elapsed > 100 ? this.#lc.info : this.#lc.debug)?.(\n              `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n            );\n          }\n\n          for (const entry of entries) {\n            if (entry.watermark === sub.watermark) {\n              // This should be the first entry.\n              // Catchup starts from *after* the watermark.\n              watermarkFound = true;\n            } else if (watermarkFound) {\n              lastBatchConsumed = sub.catchup(toDownstream(entry)).result;\n              count++;\n            } else if (mode === 'backup') {\n              throw new AutoResetSignal(\n                `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n              );\n            } else {\n              this.#lc.warn?.(\n                `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n              );\n              sub.close(\n                ErrorType.WatermarkTooOld,\n                `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n              );\n              return;\n            }\n          }\n        }\n        if (watermarkFound) {\n          await lastBatchConsumed;\n          this.#lc.info?.(\n            `caught up ${sub.id} with ${count} changes (${\n              Date.now() - start\n            } ms)`,\n          );\n        } else {\n          this.#lc.warn?.(\n            `subscriber at watermark ${sub.watermark} is ahead of latest watermark`,\n          );\n        }\n        // Flushes the backlog of messages buffered during catchup and\n        // allows the subscription to forward subsequent messages immediately.\n        sub.setCaughtUp();\n      });\n    } catch (err) {\n      this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n      if (err instanceof AutoResetSignal) {\n        await markResetRequired(this.#db, this.#shard);\n        this.#onFatal(err);\n      }\n      sub.fail(err);\n    }\n  }\n\n  stop() {\n    this.#queue.enqueue('stop');\n    return promiseVoid;\n  }\n}\n\nfunction toDownstream(entry: ChangeEntry): WatermarkedChange {\n  const {watermark, change} = entry;\n  switch (change.tag) {\n    case 'begin':\n      return [watermark, ['begin', change, {commitWatermark: watermark}]];\n    case 'commit':\n      return [watermark, ['commit', change, {watermark}]];\n    case 'rollback':\n      return [watermark, ['rollback', change]];\n    default:\n      return [watermark, ['data', change]];\n  }\n}\n"],"names":["Mode.SERIALIZABLE","tx","Mode.READONLY","start","ErrorType.WatermarkTooOld"],"mappings":";;;;;;;;;;;;;;;;AA2DA,MAAM,qCAAqC;AAgCpC,MAAM,OAA0B;AAAA,EAC5B,KAAK;AAAA,EACL;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS,IAAI,MAAA;AAAA,EAEtB,WAAW;AAAA,EAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,YAAY;AACnD,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,oBAAoB;AACzB,SAAK,qBAAqB;AAC1B,SAAK,MAAM;AACX,SAAK,kBAAkB;AACvB,SAAK,cAAc;AACnB,SAAK,WAAW;AAAA,EAClB;AAAA;AAAA,EAGA,KAAK,OAAe;AAClB,WAAO,KAAK,IAAI,GAAG,UAAU,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;AAAA,EACtD;AAAA,EAEA,MAAM,kBAAkB;AACtB,UAAM,KAAK,KAAK;AAChB,UAAM,QAAQ,KAAK;AACnB,UAAM,eAAe,KAAK;AAC1B,UAAM,gBAAgB,KAAK;AAE3B,UAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,aAAa,MAAM,YAAY;AACxC,UAAM,YAAY,KAAK,KAAK,kBAAkB,CAAC,QAAQ,GAAG,EAAC,OAAO,cAAc,oBAAA,CAAoB,CAAC;AACrG,SAAK,IAAI,OAAO,wBAAwB,mBAAmB,EAAE;AAAA,EAC/D;AAAA,EAEA,MAAM,gCAAiD;AAIrD,UAAM,EAAC,SAAS,OAAO,QAAA,IAAW,SAAA;AAClC,SAAK,OAAO,QAAQ,CAAC,SAAS,OAAO,CAAC;AACtC,UAAM;AAEN,UAAM,CAAC,EAAC,cAAA,CAAc,IAAI,MAAM,KAAK;AAAA,oCACL,KAAK,KAAK,kBAAkB,CAAC;AAC7D,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,4BAAoD;AACxD,UAAM,CAAC,EAAC,aAAA,CAAa,IAAI,MAAM,KAAK;AAAA,qDAGa,KAAK,KAAK,WAAW,CAAC;AACvE,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,WAAoC;AACrD,WAAO,KAAK,IAAI,MAAMA,cAAmB,OAAM,QAAO;AACpD,8BAAwB,GAAG;AAK3B,YAAM,CAAC,EAAC,OAAM,IAAI,MAAM;AAAA,wBACN,KAAK,KAAK,kBAAkB,CAAC;AAC/C,UAAI,UAAU,KAAK,SAAS;AAC1B,aAAK,IAAI;AAAA,UACP,sCAAsC,SAAS;AAAA,QAAA;AAEjD,eAAO;AAAA,MACT;AAEA,YAAM,CAAC,EAAC,SAAQ,IAAI,MAAM;AAAA;AAAA,wBAER,KAAK,KAAK,WAAW,CAAC,sBAAsB,SAAS;AAAA;AAAA;AAGvE,aAAO,OAAO,OAAO;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,OAA0B;AAC9B,SAAK,OAAO,QAAQ,CAAC,UAAU,KAAK,CAAC;AAAA,EACvC;AAAA,EAEA,QAAQ;AACN,SAAK,OAAO,QAAQ,CAAC,OAAO,CAAC;AAAA,EAC/B;AAAA,EAEA,OAAO,GAAkB;AACvB,SAAK,OAAO,QAAQ,CAAC;AAAA,EACvB;AAAA,EAEA,QAAQ,YAAwB,MAAsB;AACpD,SAAK,OAAO,QAAQ,CAAC,cAAc,EAAC,YAAY,KAAA,CAAK,CAAC;AAAA,EACxD;AAAA,EAEA,gBAAuC;AAAA,EAEvC,eAA0C;AACxC,QAAI,CAAC,KAAK,UAAU;AAClB,aAAO;AAAA,IACT;AACA,QACE,KAAK,kBAAkB,QACvB,KAAK,OAAO,KAAA,IAAS,oCACrB;AACA,WAAK,IAAI;AAAA,QACP,+BAA+B,KAAK,OAAO,KAAA,CAAM;AAAA,MAAA;AAEnD,WAAK,gBAAgB,SAAA;AAAA,IACvB;AACA,WAAO,KAAK,eAAe;AAAA,EAC7B;AAAA,EAEA,4BAA4B;AAC1B,QACE,KAAK,kBAAkB;AAAA,IAEvB,KAAK,OAAO,SAAS,qCAAqC,KAC1D;AACA,WAAK,IAAI;AAAA,QACP,gCAAgC,KAAK,OAAO,KAAA,CAAM;AAAA,MAAA;AAEpD,WAAK,cAAc,QAAA;AACnB,WAAK,gBAAgB;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,WAAW;AAChB,QAAI;AACF,YAAM,KAAK,cAAA;AAAA,IACb,UAAA;AACE,WAAK,WAAW;AAChB,WAAK,IAAI,OAAO,gBAAgB;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,MAAM,gBAAgB;AACpB,QAAI,KAAgC;AACpC,QAAI;AAEJ,UAAM,eAAoC,CAAA;AAC1C,YAAQ,MAAM,MAAM,KAAK,OAAO,QAAA,OAAe,QAAQ;AACrD,WAAK,0BAAA;AAEL,YAAM,CAAC,OAAO,IAAI;AAClB,cAAQ,SAAA;AAAA,QACN,KAAK,SAAS;AACZ,gBAAM,cAAc,IAAI,CAAC;AACzB,sBAAA;AACA;AAAA,QACF;AAAA,QACA,KAAK,cAAc;AACjB,gBAAM,aAAa,IAAI,CAAC;AACxB,cAAI,IAAI;AACN,yBAAa,KAAK,UAAU;AAAA,UAC9B,OAAO;AACL,kBAAM,KAAK,cAAc,CAAC,UAAU,CAAC;AAAA,UACvC;AACA;AAAA,QACF;AAAA,QACA,KAAK;AACH,eAAK,YAAY,GAAG;AACpB;AAAA,QACF,KAAK,SAAS;AACZ,cAAI,IAAI;AACN,eAAG,KAAK,MAAA;AACR,kBAAM,GAAG,KAAK,KAAA;AACd,iBAAK;AAAA,UACP;AACA;AAAA,QACF;AAAA,MAAA;AAGF,YAAM,CAAC,WAAW,UAAU,IAAI,IAAI,CAAC;AACrC,YAAM,CAAC,KAAK,MAAM,IAAI;AACtB,UAAI,QAAQ,SAAS;AACnB,eAAO,CAAC,IAAI,+CAA+C;AAC3D,cAAM,EAAC,SAAS,SAAS,OAAA,IAAU,SAAA;AACnC,aAAK;AAAA,UACH,MAAM,IAAI;AAAA,YACR,KAAK,IAAI,YAAY,aAAa,SAAS;AAAA,YAC3CA;AAAAA,UAAK;AAAA,UAEP,oBAAoB;AAAA,UACpB,KAAK;AAAA,UACL,0BAA0B;AAAA,QAAA;AAE5B,WAAG,KAAK,IAAI,KAAK,GAAG;AAGpB,aAAK,GAAG,KAAK,QAAQ,CAAAC,QAAM;AACzBA;AAAAA,0BACgB,KAAK,KAAK,kBAAkB,CAAC,GAAG;AAAA,YAC9C,CAAC,CAAC,MAAM,MAAM,QAAQ,MAAM;AAAA,YAC5B;AAAA,UAAA;AAEF,iBAAO,CAAA;AAAA,QACT,CAAC;AAAA,MACH,OAAO;AACL,eAAO,IAAI,YAAY,GAAG,yBAAyB;AACnD,WAAG;AAAA,MACL;AAEA,YAAM,QAAQ;AAAA,QACZ,WAAW,QAAQ,WAAW,YAAY,GAAG;AAAA,QAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;AAAA,QACtD,KAAK,GAAG;AAAA,QACR;AAAA,MAAA;AAGF,YAAM,YAAY,GAAG,KAAK,QAAQ,CAAAA,QAAM;AAAA,QACtCA;AAAAA,sBACc,KAAK,KAAK,WAAW,CAAC,IAAIA,IAAG,KAAK,CAAC;AAAA,MAAA,CAClD;AAED,UAAI,QAAQ,UAAU,GAAG,MAAM,QAAW,GAAG;AAI3C,cAAM;AAAA,MACR;AAEA,UAAI,QAAQ,UAAU;AACpB,cAAM,EAAC,MAAA,IAAS,MAAM,GAAG;AACzB,YAAI,UAAU,KAAK,SAAS;AAE1B,aAAG,KAAK;AAAA,YACN,IAAI,WAAW,2CAA2C,KAAK,EAAE;AAAA,UAAA;AAAA,QAErE,OAAO;AAEL,gBAAM,gBAAgB;AACtB,eAAK,GAAG,KAAK,QAAQ,CAAAA,QAAM;AAAA,YACzBA;AAAAA,qBACS,KAAK,KAAK,kBAAkB,CAAC,QAAQA,IAAG,EAAC,cAAA,CAAc,CAAC;AAAA,UAAA,CAClE;AACD,aAAG,KAAK,QAAA;AAAA,QACV;AAEA,YAAI;AACF,gBAAM,GAAG,KAAK,KAAA;AAAA,QAChB,SAAS,GAAG;AACV,cACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;AAEA,kBAAM,IAAI,WAAW,mCAAmC,EAAC,OAAO,GAAE;AAAA,UACpE;AACA,gBAAM;AAAA,QACR;AAEA,aAAK;AAGL,aAAK,YAAY,UAAU;AAI3B,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD,WAAW,QAAQ,YAAY;AAG7B,WAAG,KAAK,MAAA;AACR,cAAM,GAAG,KAAK,KAAA;AACd,aAAK;AAEL,cAAM,KAAK,cAAc,aAAa,OAAO,CAAC,CAAC;AAAA,MACjD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,MAA2B;AAC7C,QAAI,KAAK,WAAW,GAAG;AACrB;AAAA,IACF;AAEA,UAAM,SAAS,IAAI;AAAA,MACjB,KAAK,IAAI,YAAY,QAAQ,SAAS;AAAA,MACtCC;AAAAA,IAAK;AAEP,WAAO,IAAI,KAAK,GAAG;AAMnB,UAAM,OAAO,gBAAgB,MAAM;AAAA,IAAC,CAAC;AAIrC,SAAK,QAAQ,IAAI,KAAK,IAAI,CAAA,QAAO,KAAK,SAAS,KAAK,MAAM,CAAC,CAAC,EAAE;AAAA,MAAQ,MACpE,OAAO,QAAA;AAAA,IAAQ;AAAA,EAEnB;AAAA,EAEA,MAAM,SACJ,EAAC,YAAY,KAAK,KAAA,GAClB,QACA;AACA,QAAI;AACF,YAAM,OAAO,gBAAgB,OAAM,OAAM;AACvC,cAAM,QAAQ,KAAK,IAAA;AAInB,YAAI,iBAAiB,IAAI,cAAc,KAAK;AAC5C,YAAI,QAAQ;AACZ,YAAI;AAEJ,yBAAiB,WAAW;AAAA,0CACM,KAAK,KAAK,WAAW,CAAC;AAAA,gCAChC,IAAI,SAAS;AAAA,oCACT,OAAO,GAAI,GAAG;AAOxC,gBAAMC,SAAQ,YAAY,IAAA;AAC1B,gBAAM;AACN,gBAAM,UAAU,YAAY,IAAA,IAAQA;AACpC,cAAI,mBAAmB;AACrB,aAAC,UAAU,MAAM,KAAK,IAAI,OAAO,KAAK,IAAI;AAAA,cACxC,UAAU,QAAQ,QAAQ,CAAC,CAAC,WAAW,IAAI,EAAE;AAAA,YAAA;AAAA,UAEjD;AAEA,qBAAW,SAAS,SAAS;AAC3B,gBAAI,MAAM,cAAc,IAAI,WAAW;AAGrC,+BAAiB;AAAA,YACnB,WAAW,gBAAgB;AACzB,kCAAoB,IAAI,QAAQ,aAAa,KAAK,CAAC,EAAE;AACrD;AAAA,YACF,WAAW,SAAS,UAAU;AAC5B,oBAAM,IAAI;AAAA,gBACR,+BAA+B,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAAA,YAExF,OAAO;AACL,mBAAK,IAAI;AAAA,gBACP,qCAAqC,IAAI,SAAS,yBAAyB,MAAM,SAAS;AAAA,cAAA;AAE5F,kBAAI;AAAA,gBACFC;AAAAA,gBACA,mCAAmC,MAAM,SAAS,eAAe,IAAI,SAAS;AAAA,cAAA;AAEhF;AAAA,YACF;AAAA,UACF;AAAA,QACF;AACA,YAAI,gBAAgB;AAClB,gBAAM;AACN,eAAK,IAAI;AAAA,YACP,aAAa,IAAI,EAAE,SAAS,KAAK,aAC/B,KAAK,QAAQ,KACf;AAAA,UAAA;AAAA,QAEJ,OAAO;AACL,eAAK,IAAI;AAAA,YACP,2BAA2B,IAAI,SAAS;AAAA,UAAA;AAAA,QAE5C;AAGA,YAAI,YAAA;AAAA,MACN,CAAC;AAAA,IACH,SAAS,KAAK;AACZ,WAAK,IAAI,QAAQ,sCAAsC,IAAI,EAAE,IAAI,GAAG;AACpE,UAAI,eAAe,iBAAiB;AAClC,cAAM,kBAAkB,KAAK,KAAK,KAAK,MAAM;AAC7C,aAAK,SAAS,GAAG;AAAA,MACnB;AACA,UAAI,KAAK,GAAG;AAAA,IACd;AAAA,EACF;AAAA,EAEA,OAAO;AACL,SAAK,OAAO,QAAQ,MAAM;AAC1B,WAAO;AAAA,EACT;AACF;AAEA,SAAS,aAAa,OAAuC;AAC3D,QAAM,EAAC,WAAW,OAAA,IAAU;AAC5B,UAAQ,OAAO,KAAA;AAAA,IACb,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,SAAS,QAAQ,EAAC,iBAAiB,UAAA,CAAU,CAAC;AAAA,IACpE,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,UAAU,QAAQ,EAAC,UAAA,CAAU,CAAC;AAAA,IACpD,KAAK;AACH,aAAO,CAAC,WAAW,CAAC,YAAY,MAAM,CAAC;AAAA,IACzC;AACE,aAAO,CAAC,WAAW,CAAC,QAAQ,MAAM,CAAC;AAAA,EAAA;AAEzC;"}

package/out/zero-cache/src/services/mutagen/pusher.js

@@ -377,6 +377,10 @@ function assertAreCompatiblePushes(left, right) {
     left.push.pushVersion === right.push.pushVersion,
     "pushVersion must be the same for all pushes with the same clientID"
   );
+  assert(
+    left.httpCookie === right.httpCookie,
+    "httpCookie must be the same for all pushes with the same clientID"
+  );
 }
 export {
   PusherService,

package/out/zero-cache/src/services/mutagen/pusher.js.map

@@ -1 +1 @@
-{"version":3,"file":"pusher.js","sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {groupBy} from '../../../../shared/src/arrays.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {getErrorMessage} from '../../../../shared/src/error.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport type {Downstream} from '../../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../../zero-protocol/src/error-reason.ts';\nimport {\n  isProtocolError,\n  type PushFailedBody,\n} from '../../../../zero-protocol/src/error.ts';\nimport {\n  pushResponseSchema,\n  type MutationID,\n  type PushBody,\n  type PushResponse,\n} from '../../../../zero-protocol/src/push.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport {compileUrlPattern, fetchFromAPIServer} from '../../custom/fetch.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport {ProtocolErrorWithLevel} from '../../types/error-with-level.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport {upstreamSchema} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {HandlerResult, StreamResult} from '../../workers/connection.ts';\nimport type {RefCountedService, Service} from '../service.ts';\n\nexport interface Pusher extends RefCountedService {\n  readonly pushURL: string | undefined;\n\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n  ): Source<Downstream>;\n  enqueuePush(\n    clientID: string,\n    push: PushBody,\n    auth: string | undefined,\n    httpCookie: string | undefined,\n  ): HandlerResult;\n  ackMutationResponses(upToID: MutationID): Promise<void>;\n}\n\ntype Config = Pick<ZeroConfig, 'app' | 'shard'>;\n\n/**\n * Receives push messages from zero-client and forwards\n * them the the user's API server.\n *\n * If the user's API server is taking too long to process\n * the push, the PusherService will add the push to a queue\n * and send pushes in bulk the next time the user's API server\n * is available.\n *\n * - One PusherService exists per client group.\n * - Mutations for a given client are always sent in-order\n * - Mutations for different clients in the same group may be interleaved\n */\nexport class PusherService implements Service, Pusher {\n  readonly id: string;\n  readonly #pusher: PushWorker;\n  readonly #queue: Queue<PusherEntryOrStop>;\n  readonly #pushConfig: ZeroConfig['push'] & {url: string[]};\n  readonly #upstream: PostgresDB;\n  readonly #config: Config;\n  #stopped: Promise<void> | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  constructor(\n    upstream: PostgresDB,\n    appConfig: Config,\n    pushConfig: ZeroConfig['push'] & {url: string[]},\n    lc: LogContext,\n    clientGroupID: string,\n  ) {\n    this.#config = appConfig;\n    this.#upstream = upstream;\n    this.#queue = new Queue();\n    this.#pusher = new PushWorker(\n      appConfig,\n      lc,\n      pushConfig.url,\n      pushConfig.apiKey,\n      this.#queue,\n    );\n    this.id = clientGroupID;\n    this.#pushConfig = pushConfig;\n  }\n\n  get pushURL(): string | undefined {\n    return this.#pusher.pushURL[0];\n  }\n\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n  ) {\n    return this.#pusher.initConnection(clientID, wsID, userPushURL);\n  }\n\n  enqueuePush(\n    clientID: string,\n    push: PushBody,\n    auth: string | undefined,\n    httpCookie: string | undefined,\n  ): Exclude<HandlerResult, StreamResult> {\n    if (!this.#pushConfig.forwardCookies) {\n      httpCookie = undefined; // remove cookies if not forwarded\n    }\n    this.#queue.enqueue({push, auth, clientID, httpCookie});\n\n    return {\n      type: 'ok',\n    };\n  }\n\n  async ackMutationResponses(upToID: MutationID) {\n    // delete the relevant rows from the `mutations` table\n    const sql = this.#upstream;\n    await sql`DELETE FROM ${sql(\n      upstreamSchema({\n        appID: this.#config.app.id,\n        shardNum: this.#config.shard.num,\n      }),\n    )}.mutations WHERE \"clientGroupID\" = ${this.id} AND \"clientID\" = ${upToID.clientID} AND \"mutationID\" <= ${upToID.id}`;\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'PusherService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'PusherService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  run(): Promise<void> {\n    this.#stopped = this.#pusher.run();\n    return this.#stopped;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return must(this.#stopped, 'Stop was called before `run`');\n    }\n    this.#isStopped = true;\n    this.#queue.enqueue('stop');\n    return must(this.#stopped, 'Stop was called before `run`');\n  }\n}\n\ntype PusherEntry = {\n  push: PushBody;\n  auth: string | undefined;\n  httpCookie: string | undefined;\n  clientID: string;\n};\ntype PusherEntryOrStop = PusherEntry | 'stop';\n\n/**\n * Awaits items in the queue then drains and sends them all\n * to the user's API server.\n */\nclass PushWorker {\n  readonly #pushURLs: string[];\n  readonly #pushURLPatterns: URLPattern[];\n  readonly #apiKey: string | undefined;\n  readonly #queue: Queue<PusherEntryOrStop>;\n  readonly #lc: LogContext;\n  readonly #config: Config;\n  readonly #clients: Map<\n    string,\n    {\n      wsID: string;\n      downstream: Subscription<Downstream>;\n    }\n  >;\n  #userPushURL?: string | undefined;\n\n  readonly #customMutations = getOrCreateCounter(\n    'mutation',\n    'custom',\n    'Number of custom mutations processed',\n  );\n  readonly #pushes = getOrCreateCounter(\n    'mutation',\n    'pushes',\n    'Number of pushes processed by the pusher',\n  );\n\n  constructor(\n    config: Config,\n    lc: LogContext,\n    pushURL: string[],\n    apiKey: string | undefined,\n    queue: Queue<PusherEntryOrStop>,\n  ) {\n    this.#pushURLs = pushURL;\n    this.#lc = lc.withContext('component', 'pusher');\n    this.#pushURLPatterns = pushURL.map(compileUrlPattern);\n    this.#apiKey = apiKey;\n    this.#queue = queue;\n    this.#config = config;\n    this.#clients = new Map();\n  }\n\n  get pushURL() {\n    return this.#pushURLs;\n  }\n\n  /**\n   * Returns a new downstream stream if the clientID,wsID pair has not been seen before.\n   * If a clientID already exists with a different wsID, that client's downstream is cancelled.\n   */\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n  ) {\n    const existing = this.#clients.get(clientID);\n    if (existing && existing.wsID === wsID) {\n      // already initialized for this socket\n      throw new Error('Connection was already initialized');\n    }\n\n    // client is back on a new connection\n    if (existing) {\n      existing.downstream.cancel();\n    }\n\n    // Handle client group level URL parameters\n    if (this.#userPushURL === undefined) {\n      // First client in the group - store its URL\n      this.#userPushURL = userPushURL;\n    } else {\n      // Validate that subsequent clients have compatible parameters\n      if (this.#userPushURL !== userPushURL) {\n        this.#lc.warn?.(\n          'Client provided different mutate parameters than client group',\n          {\n            clientID,\n            clientURL: userPushURL,\n            clientGroupURL: this.#userPushURL,\n          },\n        );\n      }\n    }\n\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => {\n        this.#clients.delete(clientID);\n      },\n    });\n    this.#clients.set(clientID, {wsID, downstream});\n    return downstream;\n  }\n\n  async run() {\n    for (;;) {\n      const task = await this.#queue.dequeue();\n      const rest = this.#queue.drain();\n      const [pushes, terminate] = combinePushes([task, ...rest]);\n      for (const push of pushes) {\n        const response = await this.#processPush(push);\n        await this.#fanOutResponses(response);\n      }\n\n      if (terminate) {\n        break;\n      }\n    }\n  }\n\n  /**\n   * 1. If the entire `push` fails, we send the error to relevant clients.\n   * 2. If the push succeeds, we look for any mutation failure that should cause the connection to terminate\n   *  and terminate the connection for those clients.\n   */\n  #fanOutResponses(response: PushResponse) {\n    const connectionTerminations: (() => void)[] = [];\n\n    // if the entire push failed, send that to the client.\n    if ('kind' in response || 'error' in response) {\n      this.#lc.warn?.(\n        'The server behind ZERO_MUTATE_URL returned a push error.',\n        response,\n      );\n      const groupedMutationIDs = groupBy(\n        response.mutationIDs ?? [],\n        m => m.clientID,\n      );\n      for (const [clientID, mutationIDs] of groupedMutationIDs) {\n        const client = this.#clients.get(clientID);\n        if (!client) {\n          continue;\n        }\n\n        // We do not resolve mutations on the client if the push fails\n        // as those mutations will be retried.\n        if ('error' in response) {\n          // This error code path will eventually be removed when we\n          // no longer support the legacy push error format.\n          const pushFailedBody: PushFailedBody =\n            response.error === 'http'\n              ? {\n                  kind: ErrorKind.PushFailed,\n                  origin: ErrorOrigin.ZeroCache,\n                  reason: ErrorReason.HTTP,\n                  status: response.status,\n                  bodyPreview: response.details,\n                  mutationIDs,\n                  message: `Fetch from API server returned non-OK status ${response.status}`,\n                }\n              : response.error === 'unsupportedPushVersion'\n                ? {\n                    kind: ErrorKind.PushFailed,\n                    origin: ErrorOrigin.Server,\n                    reason: ErrorReason.UnsupportedPushVersion,\n                    mutationIDs,\n                    message: `Unsupported push version`,\n                  }\n                : {\n                    kind: ErrorKind.PushFailed,\n                    origin: ErrorOrigin.Server,\n                    reason: ErrorReason.Internal,\n                    mutationIDs,\n                    message:\n                      response.error === 'zeroPusher'\n                        ? response.details\n                        : response.error === 'unsupportedSchemaVersion'\n                          ? 'Unsupported schema version'\n                          : 'An unknown error occurred while pushing to the API server',\n                  };\n\n          this.#failDownstream(client.downstream, pushFailedBody);\n        } else if ('kind' in response) {\n          this.#failDownstream(client.downstream, response);\n        } else {\n          unreachable(response);\n        }\n      }\n    } else {\n      // Look for mutations results that should cause us to terminate the connection\n      const groupedMutations = groupBy(response.mutations, m => m.id.clientID);\n      for (const [clientID, mutations] of groupedMutations) {\n        const client = this.#clients.get(clientID);\n        if (!client) {\n          continue;\n        }\n\n        let failure: PushFailedBody | undefined;\n        let i = 0;\n        for (; i < mutations.length; i++) {\n          const m = mutations[i];\n          if ('error' in m.result) {\n            this.#lc.warn?.(\n              'The server behind ZERO_MUTATE_URL returned a mutation error.',\n              m.result,\n            );\n          }\n          // This error code path will eventually be removed,\n          // keeping this for backwards compatibility, but the server\n          // should now return a PushFailedBody with the mutationIDs\n          if ('error' in m.result && m.result.error === 'oooMutation') {\n            failure = {\n              kind: ErrorKind.PushFailed,\n              origin: ErrorOrigin.Server,\n              reason: ErrorReason.OutOfOrderMutation,\n              message: 'mutation was out of order',\n              details: m.result.details,\n              mutationIDs: mutations.map(m => ({\n                clientID: m.id.clientID,\n                id: m.id.id,\n              })),\n            };\n            break;\n          }\n        }\n\n        if (failure && i < mutations.length - 1) {\n          this.#lc.warn?.(\n            'push-response contains mutations after a mutation which should fatal the connection',\n          );\n        }\n\n        if (failure) {\n          connectionTerminations.push(() =>\n            this.#failDownstream(client.downstream, failure),\n          );\n        }\n      }\n    }\n\n    connectionTerminations.forEach(cb => cb());\n  }\n\n  async #processPush(entry: PusherEntry): Promise<PushResponse> {\n    this.#customMutations.add(entry.push.mutations.length, {\n      clientGroupID: entry.push.clientGroupID,\n    });\n    this.#pushes.add(1, {\n      clientGroupID: entry.push.clientGroupID,\n    });\n\n    // Record custom mutations for telemetry\n    recordMutation('custom', entry.push.mutations.length);\n\n    const url =\n      this.#userPushURL ??\n      must(this.#pushURLs[0], 'ZERO_MUTATE_URL is not set');\n\n    this.#lc.debug?.(\n      'pushing to',\n      url,\n      'with',\n      entry.push.mutations.length,\n      'mutations',\n    );\n\n    let mutationIDs: MutationID[] = [];\n\n    try {\n      mutationIDs = entry.push.mutations.map(m => ({\n        id: m.id,\n        clientID: m.clientID,\n      }));\n\n      return await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        url,\n        url === this.#userPushURL,\n        this.#pushURLPatterns,\n        {\n          appID: this.#config.app.id,\n          shardNum: this.#config.shard.num,\n        },\n        {\n          apiKey: this.#apiKey,\n          token: entry.auth,\n          cookie: entry.httpCookie,\n        },\n        entry.push,\n      );\n    } catch (e) {\n      if (isProtocolError(e) && e.errorBody.kind === ErrorKind.PushFailed) {\n        return {\n          ...e.errorBody,\n          mutationIDs,\n        } as const satisfies PushFailedBody;\n      }\n\n      return {\n        kind: ErrorKind.PushFailed,\n        origin: ErrorOrigin.ZeroCache,\n        reason: ErrorReason.Internal,\n        message: `Failed to push: ${getErrorMessage(e)}`,\n        mutationIDs,\n      } as const satisfies PushFailedBody;\n    }\n  }\n\n  #failDownstream(\n    downstream: Subscription<Downstream>,\n    errorBody: PushFailedBody,\n  ): void {\n    const logLevel = errorBody.origin === ErrorOrigin.Server ? 'warn' : 'error';\n    downstream.fail(new ProtocolErrorWithLevel(errorBody, logLevel));\n  }\n}\n\n/**\n * Pushes for different clientIDs could theoretically be interleaved.\n *\n * In order to do efficient batching to the user's API server,\n * we collect all pushes for the same clientID into a single push.\n */\nexport function combinePushes(\n  entries: readonly (PusherEntryOrStop | undefined)[],\n): [PusherEntry[], boolean] {\n  const pushesByClientID = new Map<string, PusherEntry[]>();\n\n  function collect() {\n    const ret: PusherEntry[] = [];\n    for (const entries of pushesByClientID.values()) {\n      const composite: PusherEntry = {\n        ...entries[0],\n        push: {\n          ...entries[0].push,\n          mutations: [],\n        },\n      };\n      ret.push(composite);\n      for (const entry of entries) {\n        assertAreCompatiblePushes(composite, entry);\n        composite.push.mutations.push(...entry.push.mutations);\n      }\n    }\n    return ret;\n  }\n\n  for (const entry of entries) {\n    if (entry === 'stop' || entry === undefined) {\n      return [collect(), true];\n    }\n\n    const {clientID} = entry;\n    const existing = pushesByClientID.get(clientID);\n    if (existing) {\n      existing.push(entry);\n    } else {\n      pushesByClientID.set(clientID, [entry]);\n    }\n  }\n\n  return [collect(), false] as const;\n}\n\n// These invariants should always be true for a given clientID.\n// If they are not, we have a bug in the code somewhere.\nfunction assertAreCompatiblePushes(left: PusherEntry, right: PusherEntry) {\n  assert(\n    left.clientID === right.clientID,\n    'clientID must be the same for all pushes',\n  );\n  assert(\n    left.auth === right.auth,\n    'auth must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.push.schemaVersion === right.push.schemaVersion,\n    'schemaVersion must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.push.pushVersion === right.push.pushVersion,\n    'pushVersion must be the same for all pushes with the same clientID',\n  );\n}\n"],"names":["ErrorKind.PushFailed","ErrorOrigin.ZeroCache","ErrorReason.HTTP","ErrorOrigin.Server","ErrorReason.UnsupportedPushVersion","ErrorReason.Internal","ErrorReason.OutOfOrderMutation","m","entries"],"mappings":";;;;;;;;;;;;;;;;;AAgEO,MAAM,cAAyC;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACT;AAAA,EACA,YAAY;AAAA,EACZ,aAAa;AAAA,EAEb,YACE,UACA,WACA,YACA,IACA,eACA;AACA,SAAK,UAAU;AACf,SAAK,YAAY;AACjB,SAAK,SAAS,IAAI,MAAA;AAClB,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX,WAAW;AAAA,MACX,KAAK;AAAA,IAAA;AAEP,SAAK,KAAK;AACV,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,IAAI,UAA8B;AAChC,WAAO,KAAK,QAAQ,QAAQ,CAAC;AAAA,EAC/B;AAAA,EAEA,eACE,UACA,MACA,aACA;AACA,WAAO,KAAK,QAAQ,eAAe,UAAU,MAAM,WAAW;AAAA,EAChE;AAAA,EAEA,YACE,UACA,MACA,MACA,YACsC;AACtC,QAAI,CAAC,KAAK,YAAY,gBAAgB;AACpC,mBAAa;AAAA,IACf;AACA,SAAK,OAAO,QAAQ,EAAC,MAAM,MAAM,UAAU,YAAW;AAEtD,WAAO;AAAA,MACL,MAAM;AAAA,IAAA;AAAA,EAEV;AAAA,EAEA,MAAM,qBAAqB,QAAoB;AAE7C,UAAM,MAAM,KAAK;AACjB,UAAM,kBAAkB;AAAA,MACtB,eAAe;AAAA,QACb,OAAO,KAAK,QAAQ,IAAI;AAAA,QACxB,UAAU,KAAK,QAAQ,MAAM;AAAA,MAAA,CAC9B;AAAA,IAAA,CACF,sCAAsC,KAAK,EAAE,qBAAqB,OAAO,QAAQ,wBAAwB,OAAO,EAAE;AAAA,EACrH;AAAA,EAEA,MAAM;AACJ,WAAO,CAAC,KAAK,YAAY,kCAAkC;AAC3D,MAAE,KAAK;AAAA,EACT;AAAA,EAEA,QAAQ;AACN,WAAO,CAAC,KAAK,YAAY,kCAAkC;AAC3D,MAAE,KAAK;AACP,QAAI,KAAK,aAAa,GAAG;AACvB,WAAK,KAAK,KAAA;AAAA,IACZ;AAAA,EACF;AAAA,EAEA,UAAmB;AACjB,WAAO,KAAK,YAAY;AAAA,EAC1B;AAAA,EAEA,MAAqB;AACnB,SAAK,WAAW,KAAK,QAAQ,IAAA;AAC7B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAAsB;AACpB,QAAI,KAAK,YAAY;AACnB,aAAO,KAAK,KAAK,UAAU,8BAA8B;AAAA,IAC3D;AACA,SAAK,aAAa;AAClB,SAAK,OAAO,QAAQ,MAAM;AAC1B,WAAO,KAAK,KAAK,UAAU,8BAA8B;AAAA,EAC3D;AACF;AAcA,MAAM,WAAW;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAOT;AAAA,EAES,mBAAmB;AAAA,IAC1B;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAEO,UAAU;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF,YACE,QACA,IACA,SACA,QACA,OACA;AACA,SAAK,YAAY;AACjB,SAAK,MAAM,GAAG,YAAY,aAAa,QAAQ;AAC/C,SAAK,mBAAmB,QAAQ,IAAI,iBAAiB;AACrD,SAAK,UAAU;AACf,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,+BAAe,IAAA;AAAA,EACtB;AAAA,EAEA,IAAI,UAAU;AACZ,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,eACE,UACA,MACA,aACA;AACA,UAAM,WAAW,KAAK,SAAS,IAAI,QAAQ;AAC3C,QAAI,YAAY,SAAS,SAAS,MAAM;AAEtC,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AAGA,QAAI,UAAU;AACZ,eAAS,WAAW,OAAA;AAAA,IACtB;AAGA,QAAI,KAAK,iBAAiB,QAAW;AAEnC,WAAK,eAAe;AAAA,IACtB,OAAO;AAEL,UAAI,KAAK,iBAAiB,aAAa;AACrC,aAAK,IAAI;AAAA,UACP;AAAA,UACA;AAAA,YACE;AAAA,YACA,WAAW;AAAA,YACX,gBAAgB,KAAK;AAAA,UAAA;AAAA,QACvB;AAAA,MAEJ;AAAA,IACF;AAEA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM;AACb,aAAK,SAAS,OAAO,QAAQ;AAAA,MAC/B;AAAA,IAAA,CACD;AACD,SAAK,SAAS,IAAI,UAAU,EAAC,MAAM,YAAW;AAC9C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,MAAM;AACV,eAAS;AACP,YAAM,OAAO,MAAM,KAAK,OAAO,QAAA;AAC/B,YAAM,OAAO,KAAK,OAAO,MAAA;AACzB,YAAM,CAAC,QAAQ,SAAS,IAAI,cAAc,CAAC,MAAM,GAAG,IAAI,CAAC;AACzD,iBAAW,QAAQ,QAAQ;AACzB,cAAM,WAAW,MAAM,KAAK,aAAa,IAAI;AAC7C,cAAM,KAAK,iBAAiB,QAAQ;AAAA,MACtC;AAEA,UAAI,WAAW;AACb;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,iBAAiB,UAAwB;AACvC,UAAM,yBAAyC,CAAA;AAG/C,QAAI,UAAU,YAAY,WAAW,UAAU;AAC7C,WAAK,IAAI;AAAA,QACP;AAAA,QACA;AAAA,MAAA;AAEF,YAAM,qBAAqB;AAAA,QACzB,SAAS,eAAe,CAAA;AAAA,QACxB,OAAK,EAAE;AAAA,MAAA;AAET,iBAAW,CAAC,UAAU,WAAW,KAAK,oBAAoB;AACxD,cAAM,SAAS,KAAK,SAAS,IAAI,QAAQ;AACzC,YAAI,CAAC,QAAQ;AACX;AAAA,QACF;AAIA,YAAI,WAAW,UAAU;AAGvB,gBAAM,iBACJ,SAAS,UAAU,SACf;AAAA,YACE,MAAMA;AAAAA,YACN,QAAQC;AAAAA,YACR,QAAQC;AAAAA,YACR,QAAQ,SAAS;AAAA,YACjB,aAAa,SAAS;AAAA,YACtB;AAAA,YACA,SAAS,gDAAgD,SAAS,MAAM;AAAA,UAAA,IAE1E,SAAS,UAAU,2BACjB;AAAA,YACE,MAAMF;AAAAA,YACN,QAAQG;AAAAA,YACR,QAAQC;AAAAA,YACR;AAAA,YACA,SAAS;AAAA,UAAA,IAEX;AAAA,YACE,MAAMJ;AAAAA,YACN,QAAQG;AAAAA,YACR,QAAQE;AAAAA,YACR;AAAA,YACA,SACE,SAAS,UAAU,eACf,SAAS,UACT,SAAS,UAAU,6BACjB,+BACA;AAAA,UAAA;AAGlB,eAAK,gBAAgB,OAAO,YAAY,cAAc;AAAA,QACxD,WAAW,UAAU,UAAU;AAC7B,eAAK,gBAAgB,OAAO,YAAY,QAAQ;AAAA,QAClD,OAAO;AACL,sBAAoB;AAAA,QACtB;AAAA,MACF;AAAA,IACF,OAAO;AAEL,YAAM,mBAAmB,QAAQ,SAAS,WAAW,CAAA,MAAK,EAAE,GAAG,QAAQ;AACvE,iBAAW,CAAC,UAAU,SAAS,KAAK,kBAAkB;AACpD,cAAM,SAAS,KAAK,SAAS,IAAI,QAAQ;AACzC,YAAI,CAAC,QAAQ;AACX;AAAA,QACF;AAEA,YAAI;AACJ,YAAI,IAAI;AACR,eAAO,IAAI,UAAU,QAAQ,KAAK;AAChC,gBAAM,IAAI,UAAU,CAAC;AACrB,cAAI,WAAW,EAAE,QAAQ;AACvB,iBAAK,IAAI;AAAA,cACP;AAAA,cACA,EAAE;AAAA,YAAA;AAAA,UAEN;AAIA,cAAI,WAAW,EAAE,UAAU,EAAE,OAAO,UAAU,eAAe;AAC3D,sBAAU;AAAA,cACR,MAAML;AAAAA,cACN,QAAQG;AAAAA,cACR,QAAQG;AAAAA,cACR,SAAS;AAAA,cACT,SAAS,EAAE,OAAO;AAAA,cAClB,aAAa,UAAU,IAAI,CAAAC,QAAM;AAAA,gBAC/B,UAAUA,GAAE,GAAG;AAAA,gBACf,IAAIA,GAAE,GAAG;AAAA,cAAA,EACT;AAAA,YAAA;AAEJ;AAAA,UACF;AAAA,QACF;AAEA,YAAI,WAAW,IAAI,UAAU,SAAS,GAAG;AACvC,eAAK,IAAI;AAAA,YACP;AAAA,UAAA;AAAA,QAEJ;AAEA,YAAI,SAAS;AACX,iCAAuB;AAAA,YAAK,MAC1B,KAAK,gBAAgB,OAAO,YAAY,OAAO;AAAA,UAAA;AAAA,QAEnD;AAAA,MACF;AAAA,IACF;AAEA,2BAAuB,QAAQ,CAAA,OAAM,GAAA,CAAI;AAAA,EAC3C;AAAA,EAEA,MAAM,aAAa,OAA2C;AAC5D,SAAK,iBAAiB,IAAI,MAAM,KAAK,UAAU,QAAQ;AAAA,MACrD,eAAe,MAAM,KAAK;AAAA,IAAA,CAC3B;AACD,SAAK,QAAQ,IAAI,GAAG;AAAA,MAClB,eAAe,MAAM,KAAK;AAAA,IAAA,CAC3B;AAGD,mBAAe,UAAU,MAAM,KAAK,UAAU,MAAM;AAEpD,UAAM,MACJ,KAAK,gBACL,KAAK,KAAK,UAAU,CAAC,GAAG,4BAA4B;AAEtD,SAAK,IAAI;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,MAAM,KAAK,UAAU;AAAA,MACrB;AAAA,IAAA;AAGF,QAAI,cAA4B,CAAA;AAEhC,QAAI;AACF,oBAAc,MAAM,KAAK,UAAU,IAAI,CAAA,OAAM;AAAA,QAC3C,IAAI,EAAE;AAAA,QACN,UAAU,EAAE;AAAA,MAAA,EACZ;AAEF,aAAO,MAAM;AAAA,QACX;AAAA,QACA;AAAA,QACA,KAAK;AAAA,QACL;AAAA,QACA,QAAQ,KAAK;AAAA,QACb,KAAK;AAAA,QACL;AAAA,UACE,OAAO,KAAK,QAAQ,IAAI;AAAA,UACxB,UAAU,KAAK,QAAQ,MAAM;AAAA,QAAA;AAAA,QAE/B;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,OAAO,MAAM;AAAA,UACb,QAAQ,MAAM;AAAA,QAAA;AAAA,QAEhB,MAAM;AAAA,MAAA;AAAA,IAEV,SAAS,GAAG;AACV,UAAI,gBAAgB,CAAC,KAAK,EAAE,UAAU,SAASP,YAAsB;AACnE,eAAO;AAAA,UACL,GAAG,EAAE;AAAA,UACL;AAAA,QAAA;AAAA,MAEJ;AAEA,aAAO;AAAA,QACL,MAAMA;AAAAA,QACN,QAAQC;AAAAA,QACR,QAAQI;AAAAA,QACR,SAAS,mBAAmB,gBAAgB,CAAC,CAAC;AAAA,QAC9C;AAAA,MAAA;AAAA,IAEJ;AAAA,EACF;AAAA,EAEA,gBACE,YACA,WACM;AACN,UAAM,WAAW,UAAU,WAAWF,SAAqB,SAAS;AACpE,eAAW,KAAK,IAAI,uBAAuB,WAAW,QAAQ,CAAC;AAAA,EACjE;AACF;AAQO,SAAS,cACd,SAC0B;AAC1B,QAAM,uCAAuB,IAAA;AAE7B,WAAS,UAAU;AACjB,UAAM,MAAqB,CAAA;AAC3B,eAAWK,YAAW,iBAAiB,UAAU;AAC/C,YAAM,YAAyB;AAAA,QAC7B,GAAGA,SAAQ,CAAC;AAAA,QACZ,MAAM;AAAA,UACJ,GAAGA,SAAQ,CAAC,EAAE;AAAA,UACd,WAAW,CAAA;AAAA,QAAC;AAAA,MACd;AAEF,UAAI,KAAK,SAAS;AAClB,iBAAW,SAASA,UAAS;AAC3B,kCAA0B,WAAW,KAAK;AAC1C,kBAAU,KAAK,UAAU,KAAK,GAAG,MAAM,KAAK,SAAS;AAAA,MACvD;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,aAAW,SAAS,SAAS;AAC3B,QAAI,UAAU,UAAU,UAAU,QAAW;AAC3C,aAAO,CAAC,QAAA,GAAW,IAAI;AAAA,IACzB;AAEA,UAAM,EAAC,aAAY;AACnB,UAAM,WAAW,iBAAiB,IAAI,QAAQ;AAC9C,QAAI,UAAU;AACZ,eAAS,KAAK,KAAK;AAAA,IACrB,OAAO;AACL,uBAAiB,IAAI,UAAU,CAAC,KAAK,CAAC;AAAA,IACxC;AAAA,EACF;AAEA,SAAO,CAAC,QAAA,GAAW,KAAK;AAC1B;AAIA,SAAS,0BAA0B,MAAmB,OAAoB;AACxE;AAAA,IACE,KAAK,aAAa,MAAM;AAAA,IACxB;AAAA,EAAA;AAEF;AAAA,IACE,KAAK,SAAS,MAAM;AAAA,IACpB;AAAA,EAAA;AAEF;AAAA,IACE,KAAK,KAAK,kBAAkB,MAAM,KAAK;AAAA,IACvC;AAAA,EAAA;AAEF;AAAA,IACE,KAAK,KAAK,gBAAgB,MAAM,KAAK;AAAA,IACrC;AAAA,EAAA;AAEJ;"}
+{"version":3,"file":"pusher.js","sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {groupBy} from '../../../../shared/src/arrays.ts';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {getErrorMessage} from '../../../../shared/src/error.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport type {Downstream} from '../../../../zero-protocol/src/down.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../../zero-protocol/src/error-reason.ts';\nimport {\n  isProtocolError,\n  type PushFailedBody,\n} from '../../../../zero-protocol/src/error.ts';\nimport {\n  pushResponseSchema,\n  type MutationID,\n  type PushBody,\n  type PushResponse,\n} from '../../../../zero-protocol/src/push.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport {compileUrlPattern, fetchFromAPIServer} from '../../custom/fetch.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport {ProtocolErrorWithLevel} from '../../types/error-with-level.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport {upstreamSchema} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {HandlerResult, StreamResult} from '../../workers/connection.ts';\nimport type {RefCountedService, Service} from '../service.ts';\n\nexport interface Pusher extends RefCountedService {\n  readonly pushURL: string | undefined;\n\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n  ): Source<Downstream>;\n  enqueuePush(\n    clientID: string,\n    push: PushBody,\n    auth: string | undefined,\n    httpCookie: string | undefined,\n  ): HandlerResult;\n  ackMutationResponses(upToID: MutationID): Promise<void>;\n}\n\ntype Config = Pick<ZeroConfig, 'app' | 'shard'>;\n\n/**\n * Receives push messages from zero-client and forwards\n * them the the user's API server.\n *\n * If the user's API server is taking too long to process\n * the push, the PusherService will add the push to a queue\n * and send pushes in bulk the next time the user's API server\n * is available.\n *\n * - One PusherService exists per client group.\n * - Mutations for a given client are always sent in-order\n * - Mutations for different clients in the same group may be interleaved\n */\nexport class PusherService implements Service, Pusher {\n  readonly id: string;\n  readonly #pusher: PushWorker;\n  readonly #queue: Queue<PusherEntryOrStop>;\n  readonly #pushConfig: ZeroConfig['push'] & {url: string[]};\n  readonly #upstream: PostgresDB;\n  readonly #config: Config;\n  #stopped: Promise<void> | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  constructor(\n    upstream: PostgresDB,\n    appConfig: Config,\n    pushConfig: ZeroConfig['push'] & {url: string[]},\n    lc: LogContext,\n    clientGroupID: string,\n  ) {\n    this.#config = appConfig;\n    this.#upstream = upstream;\n    this.#queue = new Queue();\n    this.#pusher = new PushWorker(\n      appConfig,\n      lc,\n      pushConfig.url,\n      pushConfig.apiKey,\n      this.#queue,\n    );\n    this.id = clientGroupID;\n    this.#pushConfig = pushConfig;\n  }\n\n  get pushURL(): string | undefined {\n    return this.#pusher.pushURL[0];\n  }\n\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n  ) {\n    return this.#pusher.initConnection(clientID, wsID, userPushURL);\n  }\n\n  enqueuePush(\n    clientID: string,\n    push: PushBody,\n    auth: string | undefined,\n    httpCookie: string | undefined,\n  ): Exclude<HandlerResult, StreamResult> {\n    if (!this.#pushConfig.forwardCookies) {\n      httpCookie = undefined; // remove cookies if not forwarded\n    }\n    this.#queue.enqueue({push, auth, clientID, httpCookie});\n\n    return {\n      type: 'ok',\n    };\n  }\n\n  async ackMutationResponses(upToID: MutationID) {\n    // delete the relevant rows from the `mutations` table\n    const sql = this.#upstream;\n    await sql`DELETE FROM ${sql(\n      upstreamSchema({\n        appID: this.#config.app.id,\n        shardNum: this.#config.shard.num,\n      }),\n    )}.mutations WHERE \"clientGroupID\" = ${this.id} AND \"clientID\" = ${upToID.clientID} AND \"mutationID\" <= ${upToID.id}`;\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'PusherService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'PusherService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  run(): Promise<void> {\n    this.#stopped = this.#pusher.run();\n    return this.#stopped;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return must(this.#stopped, 'Stop was called before `run`');\n    }\n    this.#isStopped = true;\n    this.#queue.enqueue('stop');\n    return must(this.#stopped, 'Stop was called before `run`');\n  }\n}\n\ntype PusherEntry = {\n  push: PushBody;\n  auth: string | undefined;\n  httpCookie: string | undefined;\n  clientID: string;\n};\ntype PusherEntryOrStop = PusherEntry | 'stop';\n\n/**\n * Awaits items in the queue then drains and sends them all\n * to the user's API server.\n */\nclass PushWorker {\n  readonly #pushURLs: string[];\n  readonly #pushURLPatterns: URLPattern[];\n  readonly #apiKey: string | undefined;\n  readonly #queue: Queue<PusherEntryOrStop>;\n  readonly #lc: LogContext;\n  readonly #config: Config;\n  readonly #clients: Map<\n    string,\n    {\n      wsID: string;\n      downstream: Subscription<Downstream>;\n    }\n  >;\n  #userPushURL?: string | undefined;\n\n  readonly #customMutations = getOrCreateCounter(\n    'mutation',\n    'custom',\n    'Number of custom mutations processed',\n  );\n  readonly #pushes = getOrCreateCounter(\n    'mutation',\n    'pushes',\n    'Number of pushes processed by the pusher',\n  );\n\n  constructor(\n    config: Config,\n    lc: LogContext,\n    pushURL: string[],\n    apiKey: string | undefined,\n    queue: Queue<PusherEntryOrStop>,\n  ) {\n    this.#pushURLs = pushURL;\n    this.#lc = lc.withContext('component', 'pusher');\n    this.#pushURLPatterns = pushURL.map(compileUrlPattern);\n    this.#apiKey = apiKey;\n    this.#queue = queue;\n    this.#config = config;\n    this.#clients = new Map();\n  }\n\n  get pushURL() {\n    return this.#pushURLs;\n  }\n\n  /**\n   * Returns a new downstream stream if the clientID,wsID pair has not been seen before.\n   * If a clientID already exists with a different wsID, that client's downstream is cancelled.\n   */\n  initConnection(\n    clientID: string,\n    wsID: string,\n    userPushURL: string | undefined,\n  ) {\n    const existing = this.#clients.get(clientID);\n    if (existing && existing.wsID === wsID) {\n      // already initialized for this socket\n      throw new Error('Connection was already initialized');\n    }\n\n    // client is back on a new connection\n    if (existing) {\n      existing.downstream.cancel();\n    }\n\n    // Handle client group level URL parameters\n    if (this.#userPushURL === undefined) {\n      // First client in the group - store its URL\n      this.#userPushURL = userPushURL;\n    } else {\n      // Validate that subsequent clients have compatible parameters\n      if (this.#userPushURL !== userPushURL) {\n        this.#lc.warn?.(\n          'Client provided different mutate parameters than client group',\n          {\n            clientID,\n            clientURL: userPushURL,\n            clientGroupURL: this.#userPushURL,\n          },\n        );\n      }\n    }\n\n    const downstream = Subscription.create<Downstream>({\n      cleanup: () => {\n        this.#clients.delete(clientID);\n      },\n    });\n    this.#clients.set(clientID, {wsID, downstream});\n    return downstream;\n  }\n\n  async run() {\n    for (;;) {\n      const task = await this.#queue.dequeue();\n      const rest = this.#queue.drain();\n      const [pushes, terminate] = combinePushes([task, ...rest]);\n      for (const push of pushes) {\n        const response = await this.#processPush(push);\n        await this.#fanOutResponses(response);\n      }\n\n      if (terminate) {\n        break;\n      }\n    }\n  }\n\n  /**\n   * 1. If the entire `push` fails, we send the error to relevant clients.\n   * 2. If the push succeeds, we look for any mutation failure that should cause the connection to terminate\n   *  and terminate the connection for those clients.\n   */\n  #fanOutResponses(response: PushResponse) {\n    const connectionTerminations: (() => void)[] = [];\n\n    // if the entire push failed, send that to the client.\n    if ('kind' in response || 'error' in response) {\n      this.#lc.warn?.(\n        'The server behind ZERO_MUTATE_URL returned a push error.',\n        response,\n      );\n      const groupedMutationIDs = groupBy(\n        response.mutationIDs ?? [],\n        m => m.clientID,\n      );\n      for (const [clientID, mutationIDs] of groupedMutationIDs) {\n        const client = this.#clients.get(clientID);\n        if (!client) {\n          continue;\n        }\n\n        // We do not resolve mutations on the client if the push fails\n        // as those mutations will be retried.\n        if ('error' in response) {\n          // This error code path will eventually be removed when we\n          // no longer support the legacy push error format.\n          const pushFailedBody: PushFailedBody =\n            response.error === 'http'\n              ? {\n                  kind: ErrorKind.PushFailed,\n                  origin: ErrorOrigin.ZeroCache,\n                  reason: ErrorReason.HTTP,\n                  status: response.status,\n                  bodyPreview: response.details,\n                  mutationIDs,\n                  message: `Fetch from API server returned non-OK status ${response.status}`,\n                }\n              : response.error === 'unsupportedPushVersion'\n                ? {\n                    kind: ErrorKind.PushFailed,\n                    origin: ErrorOrigin.Server,\n                    reason: ErrorReason.UnsupportedPushVersion,\n                    mutationIDs,\n                    message: `Unsupported push version`,\n                  }\n                : {\n                    kind: ErrorKind.PushFailed,\n                    origin: ErrorOrigin.Server,\n                    reason: ErrorReason.Internal,\n                    mutationIDs,\n                    message:\n                      response.error === 'zeroPusher'\n                        ? response.details\n                        : response.error === 'unsupportedSchemaVersion'\n                          ? 'Unsupported schema version'\n                          : 'An unknown error occurred while pushing to the API server',\n                  };\n\n          this.#failDownstream(client.downstream, pushFailedBody);\n        } else if ('kind' in response) {\n          this.#failDownstream(client.downstream, response);\n        } else {\n          unreachable(response);\n        }\n      }\n    } else {\n      // Look for mutations results that should cause us to terminate the connection\n      const groupedMutations = groupBy(response.mutations, m => m.id.clientID);\n      for (const [clientID, mutations] of groupedMutations) {\n        const client = this.#clients.get(clientID);\n        if (!client) {\n          continue;\n        }\n\n        let failure: PushFailedBody | undefined;\n        let i = 0;\n        for (; i < mutations.length; i++) {\n          const m = mutations[i];\n          if ('error' in m.result) {\n            this.#lc.warn?.(\n              'The server behind ZERO_MUTATE_URL returned a mutation error.',\n              m.result,\n            );\n          }\n          // This error code path will eventually be removed,\n          // keeping this for backwards compatibility, but the server\n          // should now return a PushFailedBody with the mutationIDs\n          if ('error' in m.result && m.result.error === 'oooMutation') {\n            failure = {\n              kind: ErrorKind.PushFailed,\n              origin: ErrorOrigin.Server,\n              reason: ErrorReason.OutOfOrderMutation,\n              message: 'mutation was out of order',\n              details: m.result.details,\n              mutationIDs: mutations.map(m => ({\n                clientID: m.id.clientID,\n                id: m.id.id,\n              })),\n            };\n            break;\n          }\n        }\n\n        if (failure && i < mutations.length - 1) {\n          this.#lc.warn?.(\n            'push-response contains mutations after a mutation which should fatal the connection',\n          );\n        }\n\n        if (failure) {\n          connectionTerminations.push(() =>\n            this.#failDownstream(client.downstream, failure),\n          );\n        }\n      }\n    }\n\n    connectionTerminations.forEach(cb => cb());\n  }\n\n  async #processPush(entry: PusherEntry): Promise<PushResponse> {\n    this.#customMutations.add(entry.push.mutations.length, {\n      clientGroupID: entry.push.clientGroupID,\n    });\n    this.#pushes.add(1, {\n      clientGroupID: entry.push.clientGroupID,\n    });\n\n    // Record custom mutations for telemetry\n    recordMutation('custom', entry.push.mutations.length);\n\n    const url =\n      this.#userPushURL ??\n      must(this.#pushURLs[0], 'ZERO_MUTATE_URL is not set');\n\n    this.#lc.debug?.(\n      'pushing to',\n      url,\n      'with',\n      entry.push.mutations.length,\n      'mutations',\n    );\n\n    let mutationIDs: MutationID[] = [];\n\n    try {\n      mutationIDs = entry.push.mutations.map(m => ({\n        id: m.id,\n        clientID: m.clientID,\n      }));\n\n      return await fetchFromAPIServer(\n        pushResponseSchema,\n        'push',\n        this.#lc,\n        url,\n        url === this.#userPushURL,\n        this.#pushURLPatterns,\n        {\n          appID: this.#config.app.id,\n          shardNum: this.#config.shard.num,\n        },\n        {\n          apiKey: this.#apiKey,\n          token: entry.auth,\n          cookie: entry.httpCookie,\n        },\n        entry.push,\n      );\n    } catch (e) {\n      if (isProtocolError(e) && e.errorBody.kind === ErrorKind.PushFailed) {\n        return {\n          ...e.errorBody,\n          mutationIDs,\n        } as const satisfies PushFailedBody;\n      }\n\n      return {\n        kind: ErrorKind.PushFailed,\n        origin: ErrorOrigin.ZeroCache,\n        reason: ErrorReason.Internal,\n        message: `Failed to push: ${getErrorMessage(e)}`,\n        mutationIDs,\n      } as const satisfies PushFailedBody;\n    }\n  }\n\n  #failDownstream(\n    downstream: Subscription<Downstream>,\n    errorBody: PushFailedBody,\n  ): void {\n    const logLevel = errorBody.origin === ErrorOrigin.Server ? 'warn' : 'error';\n    downstream.fail(new ProtocolErrorWithLevel(errorBody, logLevel));\n  }\n}\n\n/**\n * Pushes for different clientIDs could theoretically be interleaved.\n *\n * In order to do efficient batching to the user's API server,\n * we collect all pushes for the same clientID into a single push.\n */\nexport function combinePushes(\n  entries: readonly (PusherEntryOrStop | undefined)[],\n): [PusherEntry[], boolean] {\n  const pushesByClientID = new Map<string, PusherEntry[]>();\n\n  function collect() {\n    const ret: PusherEntry[] = [];\n    for (const entries of pushesByClientID.values()) {\n      const composite: PusherEntry = {\n        ...entries[0],\n        push: {\n          ...entries[0].push,\n          mutations: [],\n        },\n      };\n      ret.push(composite);\n      for (const entry of entries) {\n        assertAreCompatiblePushes(composite, entry);\n        composite.push.mutations.push(...entry.push.mutations);\n      }\n    }\n    return ret;\n  }\n\n  for (const entry of entries) {\n    if (entry === 'stop' || entry === undefined) {\n      return [collect(), true];\n    }\n\n    const {clientID} = entry;\n    const existing = pushesByClientID.get(clientID);\n    if (existing) {\n      existing.push(entry);\n    } else {\n      pushesByClientID.set(clientID, [entry]);\n    }\n  }\n\n  return [collect(), false] as const;\n}\n\n// These invariants should always be true for a given clientID.\n// If they are not, we have a bug in the code somewhere.\nfunction assertAreCompatiblePushes(left: PusherEntry, right: PusherEntry) {\n  assert(\n    left.clientID === right.clientID,\n    'clientID must be the same for all pushes',\n  );\n  assert(\n    left.auth === right.auth,\n    'auth must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.push.schemaVersion === right.push.schemaVersion,\n    'schemaVersion must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.push.pushVersion === right.push.pushVersion,\n    'pushVersion must be the same for all pushes with the same clientID',\n  );\n  assert(\n    left.httpCookie === right.httpCookie,\n    'httpCookie must be the same for all pushes with the same clientID',\n  );\n}\n"],"names":["ErrorKind.PushFailed","ErrorOrigin.ZeroCache","ErrorReason.HTTP","ErrorOrigin.Server","ErrorReason.UnsupportedPushVersion","ErrorReason.Internal","ErrorReason.OutOfOrderMutation","m","entries"],"mappings":";;;;;;;;;;;;;;;;;AAgEO,MAAM,cAAyC;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACT;AAAA,EACA,YAAY;AAAA,EACZ,aAAa;AAAA,EAEb,YACE,UACA,WACA,YACA,IACA,eACA;AACA,SAAK,UAAU;AACf,SAAK,YAAY;AACjB,SAAK,SAAS,IAAI,MAAA;AAClB,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX,WAAW;AAAA,MACX,KAAK;AAAA,IAAA;AAEP,SAAK,KAAK;AACV,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,IAAI,UAA8B;AAChC,WAAO,KAAK,QAAQ,QAAQ,CAAC;AAAA,EAC/B;AAAA,EAEA,eACE,UACA,MACA,aACA;AACA,WAAO,KAAK,QAAQ,eAAe,UAAU,MAAM,WAAW;AAAA,EAChE;AAAA,EAEA,YACE,UACA,MACA,MACA,YACsC;AACtC,QAAI,CAAC,KAAK,YAAY,gBAAgB;AACpC,mBAAa;AAAA,IACf;AACA,SAAK,OAAO,QAAQ,EAAC,MAAM,MAAM,UAAU,YAAW;AAEtD,WAAO;AAAA,MACL,MAAM;AAAA,IAAA;AAAA,EAEV;AAAA,EAEA,MAAM,qBAAqB,QAAoB;AAE7C,UAAM,MAAM,KAAK;AACjB,UAAM,kBAAkB;AAAA,MACtB,eAAe;AAAA,QACb,OAAO,KAAK,QAAQ,IAAI;AAAA,QACxB,UAAU,KAAK,QAAQ,MAAM;AAAA,MAAA,CAC9B;AAAA,IAAA,CACF,sCAAsC,KAAK,EAAE,qBAAqB,OAAO,QAAQ,wBAAwB,OAAO,EAAE;AAAA,EACrH;AAAA,EAEA,MAAM;AACJ,WAAO,CAAC,KAAK,YAAY,kCAAkC;AAC3D,MAAE,KAAK;AAAA,EACT;AAAA,EAEA,QAAQ;AACN,WAAO,CAAC,KAAK,YAAY,kCAAkC;AAC3D,MAAE,KAAK;AACP,QAAI,KAAK,aAAa,GAAG;AACvB,WAAK,KAAK,KAAA;AAAA,IACZ;AAAA,EACF;AAAA,EAEA,UAAmB;AACjB,WAAO,KAAK,YAAY;AAAA,EAC1B;AAAA,EAEA,MAAqB;AACnB,SAAK,WAAW,KAAK,QAAQ,IAAA;AAC7B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAAsB;AACpB,QAAI,KAAK,YAAY;AACnB,aAAO,KAAK,KAAK,UAAU,8BAA8B;AAAA,IAC3D;AACA,SAAK,aAAa;AAClB,SAAK,OAAO,QAAQ,MAAM;AAC1B,WAAO,KAAK,KAAK,UAAU,8BAA8B;AAAA,EAC3D;AACF;AAcA,MAAM,WAAW;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAOT;AAAA,EAES,mBAAmB;AAAA,IAC1B;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAEO,UAAU;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF,YACE,QACA,IACA,SACA,QACA,OACA;AACA,SAAK,YAAY;AACjB,SAAK,MAAM,GAAG,YAAY,aAAa,QAAQ;AAC/C,SAAK,mBAAmB,QAAQ,IAAI,iBAAiB;AACrD,SAAK,UAAU;AACf,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,+BAAe,IAAA;AAAA,EACtB;AAAA,EAEA,IAAI,UAAU;AACZ,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,eACE,UACA,MACA,aACA;AACA,UAAM,WAAW,KAAK,SAAS,IAAI,QAAQ;AAC3C,QAAI,YAAY,SAAS,SAAS,MAAM;AAEtC,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AAGA,QAAI,UAAU;AACZ,eAAS,WAAW,OAAA;AAAA,IACtB;AAGA,QAAI,KAAK,iBAAiB,QAAW;AAEnC,WAAK,eAAe;AAAA,IACtB,OAAO;AAEL,UAAI,KAAK,iBAAiB,aAAa;AACrC,aAAK,IAAI;AAAA,UACP;AAAA,UACA;AAAA,YACE;AAAA,YACA,WAAW;AAAA,YACX,gBAAgB,KAAK;AAAA,UAAA;AAAA,QACvB;AAAA,MAEJ;AAAA,IACF;AAEA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM;AACb,aAAK,SAAS,OAAO,QAAQ;AAAA,MAC/B;AAAA,IAAA,CACD;AACD,SAAK,SAAS,IAAI,UAAU,EAAC,MAAM,YAAW;AAC9C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,MAAM;AACV,eAAS;AACP,YAAM,OAAO,MAAM,KAAK,OAAO,QAAA;AAC/B,YAAM,OAAO,KAAK,OAAO,MAAA;AACzB,YAAM,CAAC,QAAQ,SAAS,IAAI,cAAc,CAAC,MAAM,GAAG,IAAI,CAAC;AACzD,iBAAW,QAAQ,QAAQ;AACzB,cAAM,WAAW,MAAM,KAAK,aAAa,IAAI;AAC7C,cAAM,KAAK,iBAAiB,QAAQ;AAAA,MACtC;AAEA,UAAI,WAAW;AACb;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,iBAAiB,UAAwB;AACvC,UAAM,yBAAyC,CAAA;AAG/C,QAAI,UAAU,YAAY,WAAW,UAAU;AAC7C,WAAK,IAAI;AAAA,QACP;AAAA,QACA;AAAA,MAAA;AAEF,YAAM,qBAAqB;AAAA,QACzB,SAAS,eAAe,CAAA;AAAA,QACxB,OAAK,EAAE;AAAA,MAAA;AAET,iBAAW,CAAC,UAAU,WAAW,KAAK,oBAAoB;AACxD,cAAM,SAAS,KAAK,SAAS,IAAI,QAAQ;AACzC,YAAI,CAAC,QAAQ;AACX;AAAA,QACF;AAIA,YAAI,WAAW,UAAU;AAGvB,gBAAM,iBACJ,SAAS,UAAU,SACf;AAAA,YACE,MAAMA;AAAAA,YACN,QAAQC;AAAAA,YACR,QAAQC;AAAAA,YACR,QAAQ,SAAS;AAAA,YACjB,aAAa,SAAS;AAAA,YACtB;AAAA,YACA,SAAS,gDAAgD,SAAS,MAAM;AAAA,UAAA,IAE1E,SAAS,UAAU,2BACjB;AAAA,YACE,MAAMF;AAAAA,YACN,QAAQG;AAAAA,YACR,QAAQC;AAAAA,YACR;AAAA,YACA,SAAS;AAAA,UAAA,IAEX;AAAA,YACE,MAAMJ;AAAAA,YACN,QAAQG;AAAAA,YACR,QAAQE;AAAAA,YACR;AAAA,YACA,SACE,SAAS,UAAU,eACf,SAAS,UACT,SAAS,UAAU,6BACjB,+BACA;AAAA,UAAA;AAGlB,eAAK,gBAAgB,OAAO,YAAY,cAAc;AAAA,QACxD,WAAW,UAAU,UAAU;AAC7B,eAAK,gBAAgB,OAAO,YAAY,QAAQ;AAAA,QAClD,OAAO;AACL,sBAAoB;AAAA,QACtB;AAAA,MACF;AAAA,IACF,OAAO;AAEL,YAAM,mBAAmB,QAAQ,SAAS,WAAW,CAAA,MAAK,EAAE,GAAG,QAAQ;AACvE,iBAAW,CAAC,UAAU,SAAS,KAAK,kBAAkB;AACpD,cAAM,SAAS,KAAK,SAAS,IAAI,QAAQ;AACzC,YAAI,CAAC,QAAQ;AACX;AAAA,QACF;AAEA,YAAI;AACJ,YAAI,IAAI;AACR,eAAO,IAAI,UAAU,QAAQ,KAAK;AAChC,gBAAM,IAAI,UAAU,CAAC;AACrB,cAAI,WAAW,EAAE,QAAQ;AACvB,iBAAK,IAAI;AAAA,cACP;AAAA,cACA,EAAE;AAAA,YAAA;AAAA,UAEN;AAIA,cAAI,WAAW,EAAE,UAAU,EAAE,OAAO,UAAU,eAAe;AAC3D,sBAAU;AAAA,cACR,MAAML;AAAAA,cACN,QAAQG;AAAAA,cACR,QAAQG;AAAAA,cACR,SAAS;AAAA,cACT,SAAS,EAAE,OAAO;AAAA,cAClB,aAAa,UAAU,IAAI,CAAAC,QAAM;AAAA,gBAC/B,UAAUA,GAAE,GAAG;AAAA,gBACf,IAAIA,GAAE,GAAG;AAAA,cAAA,EACT;AAAA,YAAA;AAEJ;AAAA,UACF;AAAA,QACF;AAEA,YAAI,WAAW,IAAI,UAAU,SAAS,GAAG;AACvC,eAAK,IAAI;AAAA,YACP;AAAA,UAAA;AAAA,QAEJ;AAEA,YAAI,SAAS;AACX,iCAAuB;AAAA,YAAK,MAC1B,KAAK,gBAAgB,OAAO,YAAY,OAAO;AAAA,UAAA;AAAA,QAEnD;AAAA,MACF;AAAA,IACF;AAEA,2BAAuB,QAAQ,CAAA,OAAM,GAAA,CAAI;AAAA,EAC3C;AAAA,EAEA,MAAM,aAAa,OAA2C;AAC5D,SAAK,iBAAiB,IAAI,MAAM,KAAK,UAAU,QAAQ;AAAA,MACrD,eAAe,MAAM,KAAK;AAAA,IAAA,CAC3B;AACD,SAAK,QAAQ,IAAI,GAAG;AAAA,MAClB,eAAe,MAAM,KAAK;AAAA,IAAA,CAC3B;AAGD,mBAAe,UAAU,MAAM,KAAK,UAAU,MAAM;AAEpD,UAAM,MACJ,KAAK,gBACL,KAAK,KAAK,UAAU,CAAC,GAAG,4BAA4B;AAEtD,SAAK,IAAI;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA,MAAM,KAAK,UAAU;AAAA,MACrB;AAAA,IAAA;AAGF,QAAI,cAA4B,CAAA;AAEhC,QAAI;AACF,oBAAc,MAAM,KAAK,UAAU,IAAI,CAAA,OAAM;AAAA,QAC3C,IAAI,EAAE;AAAA,QACN,UAAU,EAAE;AAAA,MAAA,EACZ;AAEF,aAAO,MAAM;AAAA,QACX;AAAA,QACA;AAAA,QACA,KAAK;AAAA,QACL;AAAA,QACA,QAAQ,KAAK;AAAA,QACb,KAAK;AAAA,QACL;AAAA,UACE,OAAO,KAAK,QAAQ,IAAI;AAAA,UACxB,UAAU,KAAK,QAAQ,MAAM;AAAA,QAAA;AAAA,QAE/B;AAAA,UACE,QAAQ,KAAK;AAAA,UACb,OAAO,MAAM;AAAA,UACb,QAAQ,MAAM;AAAA,QAAA;AAAA,QAEhB,MAAM;AAAA,MAAA;AAAA,IAEV,SAAS,GAAG;AACV,UAAI,gBAAgB,CAAC,KAAK,EAAE,UAAU,SAASP,YAAsB;AACnE,eAAO;AAAA,UACL,GAAG,EAAE;AAAA,UACL;AAAA,QAAA;AAAA,MAEJ;AAEA,aAAO;AAAA,QACL,MAAMA;AAAAA,QACN,QAAQC;AAAAA,QACR,QAAQI;AAAAA,QACR,SAAS,mBAAmB,gBAAgB,CAAC,CAAC;AAAA,QAC9C;AAAA,MAAA;AAAA,IAEJ;AAAA,EACF;AAAA,EAEA,gBACE,YACA,WACM;AACN,UAAM,WAAW,UAAU,WAAWF,SAAqB,SAAS;AACpE,eAAW,KAAK,IAAI,uBAAuB,WAAW,QAAQ,CAAC;AAAA,EACjE;AACF;AAQO,SAAS,cACd,SAC0B;AAC1B,QAAM,uCAAuB,IAAA;AAE7B,WAAS,UAAU;AACjB,UAAM,MAAqB,CAAA;AAC3B,eAAWK,YAAW,iBAAiB,UAAU;AAC/C,YAAM,YAAyB;AAAA,QAC7B,GAAGA,SAAQ,CAAC;AAAA,QACZ,MAAM;AAAA,UACJ,GAAGA,SAAQ,CAAC,EAAE;AAAA,UACd,WAAW,CAAA;AAAA,QAAC;AAAA,MACd;AAEF,UAAI,KAAK,SAAS;AAClB,iBAAW,SAASA,UAAS;AAC3B,kCAA0B,WAAW,KAAK;AAC1C,kBAAU,KAAK,UAAU,KAAK,GAAG,MAAM,KAAK,SAAS;AAAA,MACvD;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,aAAW,SAAS,SAAS;AAC3B,QAAI,UAAU,UAAU,UAAU,QAAW;AAC3C,aAAO,CAAC,QAAA,GAAW,IAAI;AAAA,IACzB;AAEA,UAAM,EAAC,aAAY;AACnB,UAAM,WAAW,iBAAiB,IAAI,QAAQ;AAC9C,QAAI,UAAU;AACZ,eAAS,KAAK,KAAK;AAAA,IACrB,OAAO;AACL,uBAAiB,IAAI,UAAU,CAAC,KAAK,CAAC;AAAA,IACxC;AAAA,EACF;AAEA,SAAO,CAAC,QAAA,GAAW,KAAK;AAC1B;AAIA,SAAS,0BAA0B,MAAmB,OAAoB;AACxE;AAAA,IACE,KAAK,aAAa,MAAM;AAAA,IACxB;AAAA,EAAA;AAEF;AAAA,IACE,KAAK,SAAS,MAAM;AAAA,IACpB;AAAA,EAAA;AAEF;AAAA,IACE,KAAK,KAAK,kBAAkB,MAAM,KAAK;AAAA,IACvC;AAAA,EAAA;AAEF;AAAA,IACE,KAAK,KAAK,gBAAgB,MAAM,KAAK;AAAA,IACrC;AAAA,EAAA;AAEF;AAAA,IACE,KAAK,eAAe,MAAM;AAAA,IAC1B;AAAA,EAAA;AAEJ;"}

package/out/zero-cache/src/services/statz.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"statz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/statz.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAM1D,OAAO,KAAK,EAAC,oBAAoB,IAAI,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAsS/E,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,iBAqBlB"}
+{"version":3,"file":"statz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/statz.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAM1D,OAAO,KAAK,EAAC,oBAAoB,IAAI,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAuS/E,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,iBAqBlB"}

package/out/zero-cache/src/services/statz.js

@@ -230,6 +230,7 @@ function osStats(out) {
       ["total mem", os.totalmem()],
       ["free mem", os.freemem()],
       ["cpus", os.cpus().length],
+      ["available parallelism", os.availableParallelism()],
       ["platform", os.platform()],
       ["arch", os.arch()],
       ["release", os.release()],

package/out/zero-cache/src/services/statz.js.map

@@ -1 +1 @@
-{"version":3,"file":"statz.js","sources":["../../../../../zero-cache/src/services/statz.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport fs from 'fs';\nimport os from 'os';\nimport type {Writable} from 'stream';\nimport {BigIntJSON} from '../../../shared/src/bigint-json.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {NormalizedZeroConfig as ZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {getShardID, upstreamSchema} from '../types/shards.ts';\n\nasync function upstreamStats(\n  lc: LogContext,\n  config: ZeroConfig,\n  out: Writable,\n) {\n  const schema = upstreamSchema(getShardID(config));\n  const sql = pgClient(lc, config.upstream.db);\n\n  out.write(header('Upstream'));\n\n  await printPgStats(\n    [\n      [\n        'num replicas',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"replicas\"`,\n      ],\n      [\n        'num clients with mutations',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"clients\"`,\n      ],\n      [\n        'num mutations processed',\n        sql`SELECT SUM(\"lastMutationID\") as \"c\" FROM ${sql(schema)}.\"clients\"`,\n      ],\n    ],\n    out,\n  );\n\n  await sql.end();\n}\n\nasync function cvrStats(lc: LogContext, config: ZeroConfig, out: Writable) {\n  out.write(header('CVR'));\n\n  const schema = upstreamSchema(getShardID(config)) + '/cvr';\n  const sql = pgClient(lc, config.cvr.db);\n\n  function numQueriesPerClientGroup(\n    active: boolean,\n  ): ReturnType<ReturnType<typeof pgClient>> {\n    const filter = active\n      ? sql`WHERE \"inactivatedAt\" IS NULL AND deleted = false`\n      : sql`WHERE \"inactivatedAt\" IS NOT NULL AND (\"inactivatedAt\" + \"ttl\") > NOW()`;\n    return sql`WITH\n    group_counts AS (\n      SELECT\n        \"clientGroupID\",\n        COUNT(*) AS num_queries\n      FROM ${sql(schema)}.\"desires\"\n      ${filter}\n      GROUP BY \"clientGroupID\"\n    ),\n    -- Count distinct clientIDs per clientGroupID\n    client_per_group_counts AS (\n      SELECT\n        \"clientGroupID\",\n        COUNT(DISTINCT \"clientID\") AS num_clients\n      FROM ${sql(schema)}.\"desires\"\n      ${filter}\n      GROUP BY \"clientGroupID\"\n    )\n    -- Combine all the information\n    SELECT\n      g.\"clientGroupID\",\n      cpg.num_clients,\n      g.num_queries\n    FROM group_counts g\n    JOIN client_per_group_counts cpg ON g.\"clientGroupID\" = cpg.\"clientGroupID\"\n    ORDER BY g.num_queries DESC;`;\n  }\n\n  await printPgStats(\n    [\n      [\n        'total num queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\"`,\n      ],\n      [\n        'num unique query hashes',\n        sql`SELECT COUNT(DISTINCT \"queryHash\") as \"c\" FROM ${sql(\n          schema,\n        )}.\"desires\"`,\n      ],\n      [\n        'num active queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"inactivatedAt\" IS NULL AND \"deleted\" = false`,\n      ],\n      [\n        'num inactive queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"inactivatedAt\" IS NOT NULL AND (\"inactivatedAt\" + \"ttl\") > NOW()`,\n      ],\n      [\n        'num deleted queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"deleted\" = true`,\n      ],\n      [\n        'fresh queries percentiles',\n        sql`WITH client_group_counts AS (\n        -- Count inactive desires per clientGroupID\n        SELECT\n          \"clientGroupID\",\n          COUNT(*) AS fresh_count\n        FROM ${sql(schema)}.\"desires\"\n        WHERE\n          (\"inactivatedAt\" IS NOT NULL\n          AND (\"inactivatedAt\" + \"ttl\") > NOW()) OR (\"inactivatedAt\" IS NULL\n          AND deleted = false)\n        GROUP BY \"clientGroupID\"\n      )\n\n      SELECT\n        percentile_cont(0.50) WITHIN GROUP (ORDER BY fresh_count) AS \"p50\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY fresh_count) AS \"p75\",\n        percentile_cont(0.90) WITHIN GROUP (ORDER BY fresh_count) AS \"p90\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY fresh_count) AS \"p95\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY fresh_count) AS \"p99\",\n        MIN(fresh_count) AS \"min\",\n        MAX(fresh_count) AS \"max\",\n        AVG(fresh_count) AS \"avg\"\n      FROM client_group_counts;`,\n      ],\n      [\n        'rows per client group percentiles',\n        sql`WITH client_group_counts AS (\n        -- Count inactive desires per clientGroupID\n        SELECT\n          \"clientGroupID\",\n          COUNT(*) AS row_count\n        FROM ${sql(schema)}.\"rows\"\n        GROUP BY \"clientGroupID\"\n      )\n      SELECT\n        percentile_cont(0.50) WITHIN GROUP (ORDER BY row_count) AS \"p50\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY row_count) AS \"p75\",\n        percentile_cont(0.90) WITHIN GROUP (ORDER BY row_count) AS \"p90\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY row_count) AS \"p95\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY row_count) AS \"p99\",\n        MIN(row_count) AS \"min\",\n        MAX(row_count) AS \"max\",\n        AVG(row_count) AS \"avg\"\n      FROM client_group_counts;`,\n      ],\n      [\n        // check for AST blowup due to DNF conversion.\n        'ast sizes',\n        sql`SELECT\n        percentile_cont(0.25) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"25th_percentile\",\n        percentile_cont(0.5) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"50th_percentile\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"75th_percentile\",\n        percentile_cont(0.9) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"90th_percentile\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"95th_percentile\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"99th_percentile\",\n        MIN(length(\"clientAST\"::text)) AS \"minimum_length\",\n        MAX(length(\"clientAST\"::text)) AS \"maximum_length\",\n        AVG(length(\"clientAST\"::text))::integer AS \"average_length\",\n        COUNT(*) AS \"total_records\"\n      FROM ${sql(schema)}.\"queries\";`,\n      ],\n      [\n        // output the hash of the largest AST\n        'biggest ast hash',\n        sql`SELECT \"queryHash\", length(\"clientAST\"::text) AS \"ast_length\"\n      FROM ${sql(schema)}.\"queries\"\n      ORDER BY length(\"clientAST\"::text) DESC\n      LIMIT 1;`,\n      ],\n      [\n        'total active queries per client and client group',\n        numQueriesPerClientGroup(true),\n      ],\n      [\n        'total inactive queries per client and client group',\n        numQueriesPerClientGroup(false),\n      ],\n      [\n        'total rows per client group',\n        sql`SELECT \"clientGroupID\", COUNT(*) as \"c\" FROM ${sql(\n          schema,\n        )}.\"rows\" GROUP BY \"clientGroupID\" ORDER BY \"c\" DESC`,\n      ],\n      [\n        'num rows per query',\n        sql`SELECT\n        k.key AS \"queryHash\",\n        COUNT(*) AS row_count\n      FROM ${sql(schema)}.\"rows\" r,\n      LATERAL jsonb_each(r.\"refCounts\") k\n      GROUP BY k.key\n      ORDER BY row_count DESC;`,\n      ],\n    ] satisfies [\n      name: string,\n      query: ReturnType<ReturnType<typeof pgClient>>,\n    ][],\n    out,\n  );\n\n  await sql.end();\n}\n\nasync function changelogStats(\n  lc: LogContext,\n  config: ZeroConfig,\n  out: Writable,\n) {\n  out.write(header('Change DB'));\n  const schema = upstreamSchema(getShardID(config)) + '/cdc';\n  const sql = pgClient(lc, config.change.db);\n\n  await printPgStats(\n    [\n      [\n        'change log size',\n        sql`SELECT COUNT(*) as \"change_log_size\" FROM ${sql(schema)}.\"changeLog\"`,\n      ],\n    ],\n    out,\n  );\n  await sql.end();\n}\n\nfunction replicaStats(lc: LogContext, config: ZeroConfig, out: Writable) {\n  out.write(header('Replica'));\n  const db = new Database(lc, config.replica.file);\n  printStats(\n    'replica',\n    [\n      ['wal checkpoint', pick(first(db.pragma('WAL_CHECKPOINT')))],\n      ['page count', pick(first(db.pragma('PAGE_COUNT')))],\n      ['page size', pick(first(db.pragma('PAGE_SIZE')))],\n      ['journal mode', pick(first(db.pragma('JOURNAL_MODE')))],\n      ['synchronous', pick(first(db.pragma('SYNCHRONOUS')))],\n      ['cache size', pick(first(db.pragma('CACHE_SIZE')))],\n      ['auto vacuum', pick(first(db.pragma('AUTO_VACUUM')))],\n      ['freelist count', pick(first(db.pragma('FREELIST_COUNT')))],\n      ['wal autocheckpoint', pick(first(db.pragma('WAL_AUTOCHECKPOINT')))],\n      ['db file stats', fs.statSync(config.replica.file)],\n    ] as const,\n    out,\n  );\n}\n\nfunction osStats(out: Writable) {\n  printStats(\n    'os',\n    [\n      ['load avg', os.loadavg()],\n      ['uptime', os.uptime()],\n      ['total mem', os.totalmem()],\n      ['free mem', os.freemem()],\n      ['cpus', os.cpus().length],\n      ['platform', os.platform()],\n      ['arch', os.arch()],\n      ['release', os.release()],\n      ['uptime', os.uptime()],\n    ] as const,\n    out,\n  );\n}\n\nasync function printPgStats(\n  pendingQueries: [\n    name: string,\n    query: ReturnType<ReturnType<typeof pgClient>>,\n  ][],\n  out: Writable,\n) {\n  const results = await Promise.all(\n    pendingQueries.map(async ([name, query]) => [name, await query]),\n  );\n  for (const [name, data] of results) {\n    out.write('\\n');\n    out.write(name);\n    out.write('\\n');\n    out.write(BigIntJSON.stringify(data, null, 2));\n  }\n}\n\nfunction printStats(\n  group: string,\n  queries: readonly [name: string, result: unknown][],\n  out: Writable,\n) {\n  out.write('\\n' + header(group));\n  for (const [name, result] of queries) {\n    out.write('\\n' + name + BigIntJSON.stringify(result, null, 2));\n  }\n}\n\nexport async function handleStatzRequest(\n  lc: LogContext,\n  config: ZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Statz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  await upstreamStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  await cvrStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  await changelogStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  replicaStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  osStats(res.raw);\n  res.raw.end();\n}\n\nfunction first(x: object[]): object {\n  return x[0];\n}\n\nfunction pick(x: object): unknown {\n  return Object.values(x)[0];\n}\n\nfunction header(name: string): string {\n  return `=== ${name} ===\\n`;\n}\n"],"names":[],"mappings":";;;;;;;;AAaA,eAAe,cACb,IACA,QACA,KACA;AACA,QAAM,SAAS,eAAe,WAAW,MAAM,CAAC;AAChD,QAAM,MAAM,SAAS,IAAI,OAAO,SAAS,EAAE;AAE3C,MAAI,MAAM,OAAO,UAAU,CAAC;AAE5B,QAAM;AAAA,IACJ;AAAA,MACE;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,+CAA+C,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,IAC5D;AAAA,IAEF;AAAA,EAAA;AAGF,QAAM,IAAI,IAAA;AACZ;AAEA,eAAe,SAAS,IAAgB,QAAoB,KAAe;AACzE,MAAI,MAAM,OAAO,KAAK,CAAC;AAEvB,QAAM,SAAS,eAAe,WAAW,MAAM,CAAC,IAAI;AACpD,QAAM,MAAM,SAAS,IAAI,OAAO,IAAI,EAAE;AAEtC,WAAS,yBACP,QACyC;AACzC,UAAM,SAAS,SACX,yDACA;AACJ,WAAO;AAAA;AAAA;AAAA;AAAA;AAAA,aAKE,IAAI,MAAM,CAAC;AAAA,QAChB,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAQD,IAAI,MAAM,CAAC;AAAA,QAChB,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWZ;AAEA,QAAM;AAAA,IACJ;AAAA,MACE;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,qDAAqD;AAAA,UACnD;AAAA,QAAA,CACD;AAAA,MAAA;AAAA,MAEH;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA;AAAA;AAAA;AAAA;AAAA;AAAA,eAKO,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,MAmBpB;AAAA,QACE;AAAA,QACA;AAAA;AAAA;AAAA;AAAA;AAAA,eAKO,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,MAcpB;AAAA;AAAA,QAEE;AAAA,QACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAWK,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAElB;AAAA;AAAA,QAEE;AAAA,QACA;AAAA,aACK,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA,MAAA;AAAA,MAIlB;AAAA,QACE;AAAA,QACA,yBAAyB,IAAI;AAAA,MAAA;AAAA,MAE/B;AAAA,QACE;AAAA,QACA,yBAAyB,KAAK;AAAA,MAAA;AAAA,MAEhC;AAAA,QACE;AAAA,QACA,mDAAmD;AAAA,UACjD;AAAA,QAAA,CACD;AAAA,MAAA;AAAA,MAEH;AAAA,QACE;AAAA,QACA;AAAA;AAAA;AAAA,aAGK,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,IAIlB;AAAA,IAKF;AAAA,EAAA;AAGF,QAAM,IAAI,IAAA;AACZ;AAEA,eAAe,eACb,IACA,QACA,KACA;AACA,MAAI,MAAM,OAAO,WAAW,CAAC;AAC7B,QAAM,SAAS,eAAe,WAAW,MAAM,CAAC,IAAI;AACpD,QAAM,MAAM,SAAS,IAAI,OAAO,OAAO,EAAE;AAEzC,QAAM;AAAA,IACJ;AAAA,MACE;AAAA,QACE;AAAA,QACA,gDAAgD,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,IAC7D;AAAA,IAEF;AAAA,EAAA;AAEF,QAAM,IAAI,IAAA;AACZ;AAEA,SAAS,aAAa,IAAgB,QAAoB,KAAe;AACvE,MAAI,MAAM,OAAO,SAAS,CAAC;AAC3B,QAAM,KAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI;AAC/C;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,kBAAkB,KAAK,MAAM,GAAG,OAAO,gBAAgB,CAAC,CAAC,CAAC;AAAA,MAC3D,CAAC,cAAc,KAAK,MAAM,GAAG,OAAO,YAAY,CAAC,CAAC,CAAC;AAAA,MACnD,CAAC,aAAa,KAAK,MAAM,GAAG,OAAO,WAAW,CAAC,CAAC,CAAC;AAAA,MACjD,CAAC,gBAAgB,KAAK,MAAM,GAAG,OAAO,cAAc,CAAC,CAAC,CAAC;AAAA,MACvD,CAAC,eAAe,KAAK,MAAM,GAAG,OAAO,aAAa,CAAC,CAAC,CAAC;AAAA,MACrD,CAAC,cAAc,KAAK,MAAM,GAAG,OAAO,YAAY,CAAC,CAAC,CAAC;AAAA,MACnD,CAAC,eAAe,KAAK,MAAM,GAAG,OAAO,aAAa,CAAC,CAAC,CAAC;AAAA,MACrD,CAAC,kBAAkB,KAAK,MAAM,GAAG,OAAO,gBAAgB,CAAC,CAAC,CAAC;AAAA,MAC3D,CAAC,sBAAsB,KAAK,MAAM,GAAG,OAAO,oBAAoB,CAAC,CAAC,CAAC;AAAA,MACnE,CAAC,iBAAiB,GAAG,SAAS,OAAO,QAAQ,IAAI,CAAC;AAAA,IAAA;AAAA,IAEpD;AAAA,EAAA;AAEJ;AAEA,SAAS,QAAQ,KAAe;AAC9B;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,YAAY,GAAG,SAAS;AAAA,MACzB,CAAC,UAAU,GAAG,QAAQ;AAAA,MACtB,CAAC,aAAa,GAAG,UAAU;AAAA,MAC3B,CAAC,YAAY,GAAG,SAAS;AAAA,MACzB,CAAC,QAAQ,GAAG,KAAA,EAAO,MAAM;AAAA,MACzB,CAAC,YAAY,GAAG,UAAU;AAAA,MAC1B,CAAC,QAAQ,GAAG,MAAM;AAAA,MAClB,CAAC,WAAW,GAAG,SAAS;AAAA,MACxB,CAAC,UAAU,GAAG,OAAA,CAAQ;AAAA,IAAA;AAAA,IAExB;AAAA,EAAA;AAEJ;AAEA,eAAe,aACb,gBAIA,KACA;AACA,QAAM,UAAU,MAAM,QAAQ;AAAA,IAC5B,eAAe,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,MAAM,KAAK,CAAC;AAAA,EAAA;AAEjE,aAAW,CAAC,MAAM,IAAI,KAAK,SAAS;AAClC,QAAI,MAAM,IAAI;AACd,QAAI,MAAM,IAAI;AACd,QAAI,MAAM,IAAI;AACd,QAAI,MAAM,WAAW,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,EAC/C;AACF;AAEA,SAAS,WACP,OACA,SACA,KACA;AACA,MAAI,MAAM,OAAO,OAAO,KAAK,CAAC;AAC9B,aAAW,CAAC,MAAM,MAAM,KAAK,SAAS;AACpC,QAAI,MAAM,OAAO,OAAO,WAAW,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC/D;AACF;AAEA,eAAsB,mBACpB,IACA,QACA,KACA,KACA;AACA,QAAM,cAAc,KAAK,GAAG;AAC5B,MAAI,CAAC,qBAAqB,IAAI,QAAQ,aAAa,IAAI,GAAG;AACxD,SAAK,IACF,KAAK,GAAG,EACR,OAAO,oBAAoB,oCAAoC,EAC/D,KAAK,cAAc;AACtB;AAAA,EACF;AAEA,QAAM,cAAc,IAAI,QAAQ,IAAI,GAAG;AACvC,MAAI,IAAI,MAAM,MAAM;AACpB,QAAM,SAAS,IAAI,QAAQ,IAAI,GAAG;AAClC,MAAI,IAAI,MAAM,MAAM;AACpB,QAAM,eAAe,IAAI,QAAQ,IAAI,GAAG;AACxC,MAAI,IAAI,MAAM,MAAM;AACpB,eAAa,IAAI,QAAQ,IAAI,GAAG;AAChC,MAAI,IAAI,MAAM,MAAM;AACpB,UAAQ,IAAI,GAAG;AACf,MAAI,IAAI,IAAA;AACV;AAEA,SAAS,MAAM,GAAqB;AAClC,SAAO,EAAE,CAAC;AACZ;AAEA,SAAS,KAAK,GAAoB;AAChC,SAAO,OAAO,OAAO,CAAC,EAAE,CAAC;AAC3B;AAEA,SAAS,OAAO,MAAsB;AACpC,SAAO,OAAO,IAAI;AAAA;AACpB;"}
+{"version":3,"file":"statz.js","sources":["../../../../../zero-cache/src/services/statz.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport fs from 'fs';\nimport os from 'os';\nimport type {Writable} from 'stream';\nimport {BigIntJSON} from '../../../shared/src/bigint-json.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {NormalizedZeroConfig as ZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {getShardID, upstreamSchema} from '../types/shards.ts';\n\nasync function upstreamStats(\n  lc: LogContext,\n  config: ZeroConfig,\n  out: Writable,\n) {\n  const schema = upstreamSchema(getShardID(config));\n  const sql = pgClient(lc, config.upstream.db);\n\n  out.write(header('Upstream'));\n\n  await printPgStats(\n    [\n      [\n        'num replicas',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"replicas\"`,\n      ],\n      [\n        'num clients with mutations',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"clients\"`,\n      ],\n      [\n        'num mutations processed',\n        sql`SELECT SUM(\"lastMutationID\") as \"c\" FROM ${sql(schema)}.\"clients\"`,\n      ],\n    ],\n    out,\n  );\n\n  await sql.end();\n}\n\nasync function cvrStats(lc: LogContext, config: ZeroConfig, out: Writable) {\n  out.write(header('CVR'));\n\n  const schema = upstreamSchema(getShardID(config)) + '/cvr';\n  const sql = pgClient(lc, config.cvr.db);\n\n  function numQueriesPerClientGroup(\n    active: boolean,\n  ): ReturnType<ReturnType<typeof pgClient>> {\n    const filter = active\n      ? sql`WHERE \"inactivatedAt\" IS NULL AND deleted = false`\n      : sql`WHERE \"inactivatedAt\" IS NOT NULL AND (\"inactivatedAt\" + \"ttl\") > NOW()`;\n    return sql`WITH\n    group_counts AS (\n      SELECT\n        \"clientGroupID\",\n        COUNT(*) AS num_queries\n      FROM ${sql(schema)}.\"desires\"\n      ${filter}\n      GROUP BY \"clientGroupID\"\n    ),\n    -- Count distinct clientIDs per clientGroupID\n    client_per_group_counts AS (\n      SELECT\n        \"clientGroupID\",\n        COUNT(DISTINCT \"clientID\") AS num_clients\n      FROM ${sql(schema)}.\"desires\"\n      ${filter}\n      GROUP BY \"clientGroupID\"\n    )\n    -- Combine all the information\n    SELECT\n      g.\"clientGroupID\",\n      cpg.num_clients,\n      g.num_queries\n    FROM group_counts g\n    JOIN client_per_group_counts cpg ON g.\"clientGroupID\" = cpg.\"clientGroupID\"\n    ORDER BY g.num_queries DESC;`;\n  }\n\n  await printPgStats(\n    [\n      [\n        'total num queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\"`,\n      ],\n      [\n        'num unique query hashes',\n        sql`SELECT COUNT(DISTINCT \"queryHash\") as \"c\" FROM ${sql(\n          schema,\n        )}.\"desires\"`,\n      ],\n      [\n        'num active queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"inactivatedAt\" IS NULL AND \"deleted\" = false`,\n      ],\n      [\n        'num inactive queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"inactivatedAt\" IS NOT NULL AND (\"inactivatedAt\" + \"ttl\") > NOW()`,\n      ],\n      [\n        'num deleted queries',\n        sql`SELECT COUNT(*) as \"c\" FROM ${sql(schema)}.\"desires\" WHERE \"deleted\" = true`,\n      ],\n      [\n        'fresh queries percentiles',\n        sql`WITH client_group_counts AS (\n        -- Count inactive desires per clientGroupID\n        SELECT\n          \"clientGroupID\",\n          COUNT(*) AS fresh_count\n        FROM ${sql(schema)}.\"desires\"\n        WHERE\n          (\"inactivatedAt\" IS NOT NULL\n          AND (\"inactivatedAt\" + \"ttl\") > NOW()) OR (\"inactivatedAt\" IS NULL\n          AND deleted = false)\n        GROUP BY \"clientGroupID\"\n      )\n\n      SELECT\n        percentile_cont(0.50) WITHIN GROUP (ORDER BY fresh_count) AS \"p50\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY fresh_count) AS \"p75\",\n        percentile_cont(0.90) WITHIN GROUP (ORDER BY fresh_count) AS \"p90\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY fresh_count) AS \"p95\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY fresh_count) AS \"p99\",\n        MIN(fresh_count) AS \"min\",\n        MAX(fresh_count) AS \"max\",\n        AVG(fresh_count) AS \"avg\"\n      FROM client_group_counts;`,\n      ],\n      [\n        'rows per client group percentiles',\n        sql`WITH client_group_counts AS (\n        -- Count inactive desires per clientGroupID\n        SELECT\n          \"clientGroupID\",\n          COUNT(*) AS row_count\n        FROM ${sql(schema)}.\"rows\"\n        GROUP BY \"clientGroupID\"\n      )\n      SELECT\n        percentile_cont(0.50) WITHIN GROUP (ORDER BY row_count) AS \"p50\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY row_count) AS \"p75\",\n        percentile_cont(0.90) WITHIN GROUP (ORDER BY row_count) AS \"p90\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY row_count) AS \"p95\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY row_count) AS \"p99\",\n        MIN(row_count) AS \"min\",\n        MAX(row_count) AS \"max\",\n        AVG(row_count) AS \"avg\"\n      FROM client_group_counts;`,\n      ],\n      [\n        // check for AST blowup due to DNF conversion.\n        'ast sizes',\n        sql`SELECT\n        percentile_cont(0.25) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"25th_percentile\",\n        percentile_cont(0.5) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"50th_percentile\",\n        percentile_cont(0.75) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"75th_percentile\",\n        percentile_cont(0.9) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"90th_percentile\",\n        percentile_cont(0.95) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"95th_percentile\",\n        percentile_cont(0.99) WITHIN GROUP (ORDER BY length(\"clientAST\"::text)) AS \"99th_percentile\",\n        MIN(length(\"clientAST\"::text)) AS \"minimum_length\",\n        MAX(length(\"clientAST\"::text)) AS \"maximum_length\",\n        AVG(length(\"clientAST\"::text))::integer AS \"average_length\",\n        COUNT(*) AS \"total_records\"\n      FROM ${sql(schema)}.\"queries\";`,\n      ],\n      [\n        // output the hash of the largest AST\n        'biggest ast hash',\n        sql`SELECT \"queryHash\", length(\"clientAST\"::text) AS \"ast_length\"\n      FROM ${sql(schema)}.\"queries\"\n      ORDER BY length(\"clientAST\"::text) DESC\n      LIMIT 1;`,\n      ],\n      [\n        'total active queries per client and client group',\n        numQueriesPerClientGroup(true),\n      ],\n      [\n        'total inactive queries per client and client group',\n        numQueriesPerClientGroup(false),\n      ],\n      [\n        'total rows per client group',\n        sql`SELECT \"clientGroupID\", COUNT(*) as \"c\" FROM ${sql(\n          schema,\n        )}.\"rows\" GROUP BY \"clientGroupID\" ORDER BY \"c\" DESC`,\n      ],\n      [\n        'num rows per query',\n        sql`SELECT\n        k.key AS \"queryHash\",\n        COUNT(*) AS row_count\n      FROM ${sql(schema)}.\"rows\" r,\n      LATERAL jsonb_each(r.\"refCounts\") k\n      GROUP BY k.key\n      ORDER BY row_count DESC;`,\n      ],\n    ] satisfies [\n      name: string,\n      query: ReturnType<ReturnType<typeof pgClient>>,\n    ][],\n    out,\n  );\n\n  await sql.end();\n}\n\nasync function changelogStats(\n  lc: LogContext,\n  config: ZeroConfig,\n  out: Writable,\n) {\n  out.write(header('Change DB'));\n  const schema = upstreamSchema(getShardID(config)) + '/cdc';\n  const sql = pgClient(lc, config.change.db);\n\n  await printPgStats(\n    [\n      [\n        'change log size',\n        sql`SELECT COUNT(*) as \"change_log_size\" FROM ${sql(schema)}.\"changeLog\"`,\n      ],\n    ],\n    out,\n  );\n  await sql.end();\n}\n\nfunction replicaStats(lc: LogContext, config: ZeroConfig, out: Writable) {\n  out.write(header('Replica'));\n  const db = new Database(lc, config.replica.file);\n  printStats(\n    'replica',\n    [\n      ['wal checkpoint', pick(first(db.pragma('WAL_CHECKPOINT')))],\n      ['page count', pick(first(db.pragma('PAGE_COUNT')))],\n      ['page size', pick(first(db.pragma('PAGE_SIZE')))],\n      ['journal mode', pick(first(db.pragma('JOURNAL_MODE')))],\n      ['synchronous', pick(first(db.pragma('SYNCHRONOUS')))],\n      ['cache size', pick(first(db.pragma('CACHE_SIZE')))],\n      ['auto vacuum', pick(first(db.pragma('AUTO_VACUUM')))],\n      ['freelist count', pick(first(db.pragma('FREELIST_COUNT')))],\n      ['wal autocheckpoint', pick(first(db.pragma('WAL_AUTOCHECKPOINT')))],\n      ['db file stats', fs.statSync(config.replica.file)],\n    ] as const,\n    out,\n  );\n}\n\nfunction osStats(out: Writable) {\n  printStats(\n    'os',\n    [\n      ['load avg', os.loadavg()],\n      ['uptime', os.uptime()],\n      ['total mem', os.totalmem()],\n      ['free mem', os.freemem()],\n      ['cpus', os.cpus().length],\n      ['available parallelism', os.availableParallelism()],\n      ['platform', os.platform()],\n      ['arch', os.arch()],\n      ['release', os.release()],\n      ['uptime', os.uptime()],\n    ] as const,\n    out,\n  );\n}\n\nasync function printPgStats(\n  pendingQueries: [\n    name: string,\n    query: ReturnType<ReturnType<typeof pgClient>>,\n  ][],\n  out: Writable,\n) {\n  const results = await Promise.all(\n    pendingQueries.map(async ([name, query]) => [name, await query]),\n  );\n  for (const [name, data] of results) {\n    out.write('\\n');\n    out.write(name);\n    out.write('\\n');\n    out.write(BigIntJSON.stringify(data, null, 2));\n  }\n}\n\nfunction printStats(\n  group: string,\n  queries: readonly [name: string, result: unknown][],\n  out: Writable,\n) {\n  out.write('\\n' + header(group));\n  for (const [name, result] of queries) {\n    out.write('\\n' + name + BigIntJSON.stringify(result, null, 2));\n  }\n}\n\nexport async function handleStatzRequest(\n  lc: LogContext,\n  config: ZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Statz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  await upstreamStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  await cvrStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  await changelogStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  replicaStats(lc, config, res.raw);\n  res.raw.write('\\n\\n');\n  osStats(res.raw);\n  res.raw.end();\n}\n\nfunction first(x: object[]): object {\n  return x[0];\n}\n\nfunction pick(x: object): unknown {\n  return Object.values(x)[0];\n}\n\nfunction header(name: string): string {\n  return `=== ${name} ===\\n`;\n}\n"],"names":[],"mappings":";;;;;;;;AAaA,eAAe,cACb,IACA,QACA,KACA;AACA,QAAM,SAAS,eAAe,WAAW,MAAM,CAAC;AAChD,QAAM,MAAM,SAAS,IAAI,OAAO,SAAS,EAAE;AAE3C,MAAI,MAAM,OAAO,UAAU,CAAC;AAE5B,QAAM;AAAA,IACJ;AAAA,MACE;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,+CAA+C,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,IAC5D;AAAA,IAEF;AAAA,EAAA;AAGF,QAAM,IAAI,IAAA;AACZ;AAEA,eAAe,SAAS,IAAgB,QAAoB,KAAe;AACzE,MAAI,MAAM,OAAO,KAAK,CAAC;AAEvB,QAAM,SAAS,eAAe,WAAW,MAAM,CAAC,IAAI;AACpD,QAAM,MAAM,SAAS,IAAI,OAAO,IAAI,EAAE;AAEtC,WAAS,yBACP,QACyC;AACzC,UAAM,SAAS,SACX,yDACA;AACJ,WAAO;AAAA;AAAA;AAAA;AAAA;AAAA,aAKE,IAAI,MAAM,CAAC;AAAA,QAChB,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAQD,IAAI,MAAM,CAAC;AAAA,QAChB,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWZ;AAEA,QAAM;AAAA,IACJ;AAAA,MACE;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,qDAAqD;AAAA,UACnD;AAAA,QAAA,CACD;AAAA,MAAA;AAAA,MAEH;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA,kCAAkC,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAE/C;AAAA,QACE;AAAA,QACA;AAAA;AAAA;AAAA;AAAA;AAAA,eAKO,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,MAmBpB;AAAA,QACE;AAAA,QACA;AAAA;AAAA;AAAA;AAAA;AAAA,eAKO,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,MAcpB;AAAA;AAAA,QAEE;AAAA,QACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAWK,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,MAElB;AAAA;AAAA,QAEE;AAAA,QACA;AAAA,aACK,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA,MAAA;AAAA,MAIlB;AAAA,QACE;AAAA,QACA,yBAAyB,IAAI;AAAA,MAAA;AAAA,MAE/B;AAAA,QACE;AAAA,QACA,yBAAyB,KAAK;AAAA,MAAA;AAAA,MAEhC;AAAA,QACE;AAAA,QACA,mDAAmD;AAAA,UACjD;AAAA,QAAA,CACD;AAAA,MAAA;AAAA,MAEH;AAAA,QACE;AAAA,QACA;AAAA;AAAA;AAAA,aAGK,IAAI,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA,MAAA;AAAA,IAIlB;AAAA,IAKF;AAAA,EAAA;AAGF,QAAM,IAAI,IAAA;AACZ;AAEA,eAAe,eACb,IACA,QACA,KACA;AACA,MAAI,MAAM,OAAO,WAAW,CAAC;AAC7B,QAAM,SAAS,eAAe,WAAW,MAAM,CAAC,IAAI;AACpD,QAAM,MAAM,SAAS,IAAI,OAAO,OAAO,EAAE;AAEzC,QAAM;AAAA,IACJ;AAAA,MACE;AAAA,QACE;AAAA,QACA,gDAAgD,IAAI,MAAM,CAAC;AAAA,MAAA;AAAA,IAC7D;AAAA,IAEF;AAAA,EAAA;AAEF,QAAM,IAAI,IAAA;AACZ;AAEA,SAAS,aAAa,IAAgB,QAAoB,KAAe;AACvE,MAAI,MAAM,OAAO,SAAS,CAAC;AAC3B,QAAM,KAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI;AAC/C;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,kBAAkB,KAAK,MAAM,GAAG,OAAO,gBAAgB,CAAC,CAAC,CAAC;AAAA,MAC3D,CAAC,cAAc,KAAK,MAAM,GAAG,OAAO,YAAY,CAAC,CAAC,CAAC;AAAA,MACnD,CAAC,aAAa,KAAK,MAAM,GAAG,OAAO,WAAW,CAAC,CAAC,CAAC;AAAA,MACjD,CAAC,gBAAgB,KAAK,MAAM,GAAG,OAAO,cAAc,CAAC,CAAC,CAAC;AAAA,MACvD,CAAC,eAAe,KAAK,MAAM,GAAG,OAAO,aAAa,CAAC,CAAC,CAAC;AAAA,MACrD,CAAC,cAAc,KAAK,MAAM,GAAG,OAAO,YAAY,CAAC,CAAC,CAAC;AAAA,MACnD,CAAC,eAAe,KAAK,MAAM,GAAG,OAAO,aAAa,CAAC,CAAC,CAAC;AAAA,MACrD,CAAC,kBAAkB,KAAK,MAAM,GAAG,OAAO,gBAAgB,CAAC,CAAC,CAAC;AAAA,MAC3D,CAAC,sBAAsB,KAAK,MAAM,GAAG,OAAO,oBAAoB,CAAC,CAAC,CAAC;AAAA,MACnE,CAAC,iBAAiB,GAAG,SAAS,OAAO,QAAQ,IAAI,CAAC;AAAA,IAAA;AAAA,IAEpD;AAAA,EAAA;AAEJ;AAEA,SAAS,QAAQ,KAAe;AAC9B;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,YAAY,GAAG,SAAS;AAAA,MACzB,CAAC,UAAU,GAAG,QAAQ;AAAA,MACtB,CAAC,aAAa,GAAG,UAAU;AAAA,MAC3B,CAAC,YAAY,GAAG,SAAS;AAAA,MACzB,CAAC,QAAQ,GAAG,KAAA,EAAO,MAAM;AAAA,MACzB,CAAC,yBAAyB,GAAG,sBAAsB;AAAA,MACnD,CAAC,YAAY,GAAG,UAAU;AAAA,MAC1B,CAAC,QAAQ,GAAG,MAAM;AAAA,MAClB,CAAC,WAAW,GAAG,SAAS;AAAA,MACxB,CAAC,UAAU,GAAG,OAAA,CAAQ;AAAA,IAAA;AAAA,IAExB;AAAA,EAAA;AAEJ;AAEA,eAAe,aACb,gBAIA,KACA;AACA,QAAM,UAAU,MAAM,QAAQ;AAAA,IAC5B,eAAe,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,MAAM,KAAK,CAAC;AAAA,EAAA;AAEjE,aAAW,CAAC,MAAM,IAAI,KAAK,SAAS;AAClC,QAAI,MAAM,IAAI;AACd,QAAI,MAAM,IAAI;AACd,QAAI,MAAM,IAAI;AACd,QAAI,MAAM,WAAW,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,EAC/C;AACF;AAEA,SAAS,WACP,OACA,SACA,KACA;AACA,MAAI,MAAM,OAAO,OAAO,KAAK,CAAC;AAC9B,aAAW,CAAC,MAAM,MAAM,KAAK,SAAS;AACpC,QAAI,MAAM,OAAO,OAAO,WAAW,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC/D;AACF;AAEA,eAAsB,mBACpB,IACA,QACA,KACA,KACA;AACA,QAAM,cAAc,KAAK,GAAG;AAC5B,MAAI,CAAC,qBAAqB,IAAI,QAAQ,aAAa,IAAI,GAAG;AACxD,SAAK,IACF,KAAK,GAAG,EACR,OAAO,oBAAoB,oCAAoC,EAC/D,KAAK,cAAc;AACtB;AAAA,EACF;AAEA,QAAM,cAAc,IAAI,QAAQ,IAAI,GAAG;AACvC,MAAI,IAAI,MAAM,MAAM;AACpB,QAAM,SAAS,IAAI,QAAQ,IAAI,GAAG;AAClC,MAAI,IAAI,MAAM,MAAM;AACpB,QAAM,eAAe,IAAI,QAAQ,IAAI,GAAG;AACxC,MAAI,IAAI,MAAM,MAAM;AACpB,eAAa,IAAI,QAAQ,IAAI,GAAG;AAChC,MAAI,IAAI,MAAM,MAAM;AACpB,UAAQ,IAAI,GAAG;AACf,MAAI,IAAI,IAAA;AACV;AAEA,SAAS,MAAM,GAAqB;AAClC,SAAO,EAAE,CAAC;AACZ;AAEA,SAAS,KAAK,GAAoB;AAChC,SAAO,OAAO,OAAO,CAAC,EAAE,CAAC;AAC3B;AAEA,SAAS,OAAO,MAAsB;AACpC,SAAO,OAAO,IAAI;AAAA;AACpB;"}

package/out/zero-client/src/client/error.js

@@ -65,9 +65,9 @@ function getErrorConnectionTransition(ex) {
       case PingTimeout:
       case PullTimeout:
       case Hidden:
+      case UnexpectedBaseCookie:
         return { status: NO_STATUS_TRANSITION, reason: ex };
       // Fatal errors that should transition to error state
-      case UnexpectedBaseCookie:
       case Internal:
       case InvalidMessage:
       case UserDisconnect:

package/out/zero-client/src/client/error.js.map

@@ -1 +1 @@
-{"version":3,"file":"error.js","sources":["../../../../../zero-client/src/client/error.ts"],"sourcesContent":["import {unreachable} from '../../../shared/src/asserts.ts';\nimport {getErrorMessage} from '../../../shared/src/error.ts';\nimport type {Expand} from '../../../shared/src/expand.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../zero-protocol/src/error-reason.ts';\nimport type {ProtocolError} from '../../../zero-protocol/src/error.ts';\nimport {\n  type BackoffBody,\n  type ErrorBody,\n  isProtocolError,\n  type PushFailedBody,\n  type TransformFailedBody,\n} from '../../../zero-protocol/src/error.ts';\nimport {ClientErrorKind} from './client-error-kind.ts';\nimport {ConnectionStatus} from './connection-status.ts';\n\nexport type AuthError = ProtocolError<NeedsAuthReason>;\nexport type ClientErrorBody = {\n  kind: ClientErrorKind;\n  origin: typeof ErrorOrigin.Client;\n  message: string;\n};\nexport type ClosedError = ClientError<{\n  kind: ClientErrorKind.ClientClosed;\n  message: string;\n}>;\nexport type NeedsAuthReason = Expand<\n  | (ErrorBody & {\n      kind: ErrorKind.AuthInvalidated | ErrorKind.Unauthorized;\n    })\n  | (Extract<PushFailedBody, {reason: ErrorReason.HTTP}> & {status: 401 | 403})\n  | (Extract<TransformFailedBody, {reason: ErrorReason.HTTP}> & {\n      status: 401 | 403;\n    })\n>;\nexport type OfflineError = ClientError<{\n  kind: ClientErrorKind.Offline;\n  message: string;\n}>;\nexport type NoSocketOriginError = ClientError<{\n  kind: ClientErrorKind.NoSocketOrigin;\n  message: string;\n}>;\nexport type DisconnectedReason = OfflineError | NoSocketOriginError;\nexport type ServerError = ProtocolError<ErrorBody>;\nexport type ZeroError = ServerError | ClientError;\nexport type ZeroErrorBody = Expand<ErrorBody | ClientErrorBody>;\nexport type ZeroErrorDetails = Expand<Omit<ZeroErrorBody, 'message'>>;\nexport type ZeroErrorKind = Expand<ErrorKind | ClientErrorKind>;\n\n/**\n * Represents an error encountered by the Zero client.\n */\nexport class ClientError<\n  const T extends Omit<ClientErrorBody, 'origin'> = Omit<\n    ClientErrorBody,\n    'origin'\n  >,\n> extends Error {\n  readonly errorBody: {origin: typeof ErrorOrigin.Client} & T;\n\n  constructor(errorBody: T, options?: ErrorOptions) {\n    super(errorBody.message, options);\n    this.name = 'ClientError';\n    this.errorBody = {...errorBody, origin: ErrorOrigin.Client};\n  }\n\n  get kind(): T['kind'] {\n    return this.errorBody.kind;\n  }\n}\n\nexport function isZeroError(ex: unknown): ex is ZeroError {\n  return isClientError(ex) || isServerError(ex);\n}\n\nexport function isClientError(ex: unknown): ex is ClientError<ClientErrorBody> {\n  return (\n    ex instanceof ClientError && ex.errorBody.origin === ErrorOrigin.Client\n  );\n}\n\nexport function isServerError(ex: unknown): ex is ServerError {\n  return (\n    isProtocolError(ex) &&\n    (ex.errorBody.origin === ErrorOrigin.Server ||\n      ex.errorBody.origin === ErrorOrigin.ZeroCache)\n  );\n}\n\nexport function isOfflineError(ex: unknown): ex is OfflineError {\n  return isClientError(ex) && ex.kind === ClientErrorKind.Offline;\n}\n\nexport function isAuthError(ex: unknown): ex is AuthError {\n  if (isServerError(ex)) {\n    if (\n      ex.kind === ErrorKind.AuthInvalidated ||\n      ex.kind === ErrorKind.Unauthorized\n    ) {\n      return true;\n    }\n    if (\n      (ex.errorBody.kind === ErrorKind.PushFailed ||\n        ex.errorBody.kind === ErrorKind.TransformFailed) &&\n      ex.errorBody.reason === ErrorReason.HTTP &&\n      (ex.errorBody.status === 401 || ex.errorBody.status === 403)\n    ) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nexport function getBackoffParams(error: ZeroError): BackoffBody | undefined {\n  if (isServerError(error)) {\n    switch (error.errorBody.kind) {\n      case ErrorKind.Rebalance:\n      case ErrorKind.Rehome:\n      case ErrorKind.ServerOverloaded:\n        return error.errorBody;\n    }\n  }\n  return undefined;\n}\n\nexport const NO_STATUS_TRANSITION = 'NO_STATUS_TRANSITION';\n\nexport type ErrorConnectionTransition =\n  | {status: typeof NO_STATUS_TRANSITION; reason: ZeroError}\n  | {status: ConnectionStatus.NeedsAuth; reason: AuthError}\n  | {status: ConnectionStatus.Error; reason: ZeroError}\n  | {status: ConnectionStatus.Disconnected; reason: DisconnectedReason}\n  | {status: ConnectionStatus.Closed; reason: ZeroError};\n\n/**\n * Returns the status to transition to, or null if the error\n * indicates that the connection should continue in the current state.\n */\nexport function getErrorConnectionTransition(\n  ex: unknown,\n): ErrorConnectionTransition {\n  // Handle auth errors by transitioning to needs-auth state\n  if (isAuthError(ex)) {\n    return {\n      status: ConnectionStatus.NeedsAuth,\n      reason: ex,\n    } as const;\n  }\n\n  if (isClientError(ex)) {\n    switch (ex.kind) {\n      // Connecting errors that should continue in the current state\n      case ClientErrorKind.AbruptClose:\n      case ClientErrorKind.CleanClose:\n      case ClientErrorKind.ConnectTimeout:\n      case ClientErrorKind.PingTimeout:\n      case ClientErrorKind.PullTimeout:\n      case ClientErrorKind.Hidden:\n        return {status: NO_STATUS_TRANSITION, reason: ex} as const;\n\n      // Fatal errors that should transition to error state\n      case ClientErrorKind.UnexpectedBaseCookie:\n      case ClientErrorKind.Internal:\n      case ClientErrorKind.InvalidMessage:\n      case ClientErrorKind.UserDisconnect:\n        return {status: ConnectionStatus.Error, reason: ex} as const;\n\n      // Disconnected error (this should already result in a disconnected state)\n      case ClientErrorKind.Offline:\n      case ClientErrorKind.NoSocketOrigin:\n        return {\n          status: ConnectionStatus.Disconnected,\n          reason: ex as DisconnectedReason,\n        } as const;\n\n      // Closed error (this should already result in a closed state)\n      case ClientErrorKind.ClientClosed:\n        return {status: ConnectionStatus.Closed, reason: ex} as const;\n\n      default:\n        unreachable(ex.kind);\n    }\n  }\n\n  if (isServerError(ex)) {\n    switch (ex.kind) {\n      // Errors that should transition to error state\n      case ErrorKind.ClientNotFound:\n      case ErrorKind.InvalidConnectionRequest:\n      case ErrorKind.InvalidConnectionRequestBaseCookie:\n      case ErrorKind.InvalidConnectionRequestLastMutationID:\n      case ErrorKind.InvalidConnectionRequestClientDeleted:\n      case ErrorKind.InvalidMessage:\n      case ErrorKind.InvalidPush:\n      case ErrorKind.VersionNotSupported:\n      case ErrorKind.SchemaVersionNotSupported:\n      case ErrorKind.Internal:\n      // PushFailed and TransformFailed can be auth errors (401/403)\n      // or other errors - handle non-auth cases here\n      case ErrorKind.PushFailed:\n      case ErrorKind.TransformFailed:\n        return {status: ConnectionStatus.Error, reason: ex} as const;\n\n      // Errors that should continue with backoff/retry\n      case ErrorKind.Rebalance:\n      case ErrorKind.Rehome:\n      case ErrorKind.ServerOverloaded:\n        return {status: NO_STATUS_TRANSITION, reason: ex} as const;\n\n      // Auth errors are handled above by isAuthError check\n      case ErrorKind.AuthInvalidated:\n      case ErrorKind.Unauthorized:\n        return {\n          status: ConnectionStatus.NeedsAuth,\n          reason: ex as AuthError,\n        } as const;\n\n      // Mutation-specific errors don't affect connection state\n      case ErrorKind.MutationRateLimited:\n      case ErrorKind.MutationFailed:\n        return {status: NO_STATUS_TRANSITION, reason: ex} as const;\n\n      default:\n        unreachable(ex.kind);\n    }\n  }\n\n  // we default to error state if we don't know what to do\n  // this is a catch-all for unexpected errors\n  return {\n    status: ConnectionStatus.Error,\n    reason: new ClientError(\n      {\n        kind: ClientErrorKind.Internal,\n        message: 'Unexpected internal error: ' + getErrorMessage(ex),\n      },\n      {cause: ex},\n    ),\n  } as const;\n}\n"],"names":["ErrorOrigin.Client","ErrorOrigin.Server","ErrorOrigin.ZeroCache","ErrorKind.AuthInvalidated","ErrorKind.Unauthorized","ErrorKind.PushFailed","ErrorKind.TransformFailed","ErrorReason.HTTP","ErrorKind.Rebalance","ErrorKind.Rehome","ErrorKind.ServerOverloaded","ConnectionStatus.NeedsAuth","ClientErrorKind.AbruptClose","ClientErrorKind.CleanClose","ClientErrorKind.ConnectTimeout","ClientErrorKind.PingTimeout","ClientErrorKind.PullTimeout","ClientErrorKind.Hidden","ClientErrorKind.UnexpectedBaseCookie","ClientErrorKind.Internal","ClientErrorKind.InvalidMessage","ClientErrorKind.UserDisconnect","ConnectionStatus.Error","ClientErrorKind.Offline","ClientErrorKind.NoSocketOrigin","ConnectionStatus.Disconnected","ClientErrorKind.ClientClosed","ConnectionStatus.Closed","ErrorKind.ClientNotFound","ErrorKind.InvalidConnectionRequest","ErrorKind.InvalidConnectionRequestBaseCookie","ErrorKind.InvalidConnectionRequestLastMutationID","ErrorKind.InvalidConnectionRequestClientDeleted","ErrorKind.InvalidMessage","ErrorKind.InvalidPush","ErrorKind.VersionNotSupported","ErrorKind.SchemaVersionNotSupported","ErrorKind.Internal","ErrorKind.MutationRateLimited","ErrorKind.MutationFailed"],"mappings":";;;;;;;;AAsDO,MAAM,oBAKH,MAAM;AAAA,EACL;AAAA,EAET,YAAY,WAAc,SAAwB;AAChD,UAAM,UAAU,SAAS,OAAO;AAChC,SAAK,OAAO;AACZ,SAAK,YAAY,EAAC,GAAG,WAAW,QAAQA,OAAY;AAAA,EACtD;AAAA,EAEA,IAAI,OAAkB;AACpB,WAAO,KAAK,UAAU;AAAA,EACxB;AACF;AAEO,SAAS,YAAY,IAA8B;AACxD,SAAO,cAAc,EAAE,KAAK,cAAc,EAAE;AAC9C;AAEO,SAAS,cAAc,IAAiD;AAC7E,SACE,cAAc,eAAe,GAAG,UAAU,WAAWA;AAEzD;AAEO,SAAS,cAAc,IAAgC;AAC5D,SACE,gBAAgB,EAAE,MACjB,GAAG,UAAU,WAAWC,UACvB,GAAG,UAAU,WAAWC;AAE9B;AAMO,SAAS,YAAY,IAA8B;AACxD,MAAI,cAAc,EAAE,GAAG;AACrB,QACE,GAAG,SAASC,mBACZ,GAAG,SAASC,cACZ;AACA,aAAO;AAAA,IACT;AACA,SACG,GAAG,UAAU,SAASC,cACrB,GAAG,UAAU,SAASC,oBACxB,GAAG,UAAU,WAAWC,SACvB,GAAG,UAAU,WAAW,OAAO,GAAG,UAAU,WAAW,MACxD;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAEO,SAAS,iBAAiB,OAA2C;AAC1E,MAAI,cAAc,KAAK,GAAG;AACxB,YAAQ,MAAM,UAAU,MAAA;AAAA,MACtB,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AACH,eAAO,MAAM;AAAA,IAAA;AAAA,EAEnB;AACA,SAAO;AACT;AAEO,MAAM,uBAAuB;AAa7B,SAAS,6BACd,IAC2B;AAE3B,MAAI,YAAY,EAAE,GAAG;AACnB,WAAO;AAAA,MACL,QAAQC;AAAAA,MACR,QAAQ;AAAA,IAAA;AAAA,EAEZ;AAEA,MAAI,cAAc,EAAE,GAAG;AACrB,YAAQ,GAAG,MAAA;AAAA;AAAA,MAET,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AACH,eAAO,EAAC,QAAQ,sBAAsB,QAAQ,GAAA;AAAA;AAAA,MAGhD,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AACH,eAAO,EAAC,QAAQC,SAAwB,QAAQ,GAAA;AAAA;AAAA,MAGlD,KAAKC;AAAAA,MACL,KAAKC;AACH,eAAO;AAAA,UACL,QAAQC;AAAAA,UACR,QAAQ;AAAA,QAAA;AAAA;AAAA,MAIZ,KAAKC;AACH,eAAO,EAAC,QAAQC,QAAyB,QAAQ,GAAA;AAAA,MAEnD;AACE,oBAAY,GAAG,IAAI;AAAA,IAAA;AAAA,EAEzB;AAEA,MAAI,cAAc,EAAE,GAAG;AACrB,YAAQ,GAAG,MAAA;AAAA;AAAA,MAET,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA;AAAAA;AAAAA,MAGL,KAAKhC;AAAAA,MACL,KAAKC;AACH,eAAO,EAAC,QAAQgB,SAAwB,QAAQ,GAAA;AAAA;AAAA,MAGlD,KAAKd;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AACH,eAAO,EAAC,QAAQ,sBAAsB,QAAQ,GAAA;AAAA;AAAA,MAGhD,KAAKP;AAAAA,MACL,KAAKC;AACH,eAAO;AAAA,UACL,QAAQO;AAAAA,UACR,QAAQ;AAAA,QAAA;AAAA;AAAA,MAIZ,KAAK2B;AAAAA,MACL,KAAKC;AACH,eAAO,EAAC,QAAQ,sBAAsB,QAAQ,GAAA;AAAA,MAEhD;AACE,oBAAY,GAAG,IAAI;AAAA,IAAA;AAAA,EAEzB;AAIA,SAAO;AAAA,IACL,QAAQjB;AAAAA,IACR,QAAQ,IAAI;AAAA,MACV;AAAA,QACE,MAAMH;AAAAA,QACN,SAAS,gCAAgC,gBAAgB,EAAE;AAAA,MAAA;AAAA,MAE7D,EAAC,OAAO,GAAA;AAAA,IAAE;AAAA,EACZ;AAEJ;"}
+{"version":3,"file":"error.js","sources":["../../../../../zero-client/src/client/error.ts"],"sourcesContent":["import {unreachable} from '../../../shared/src/asserts.ts';\nimport {getErrorMessage} from '../../../shared/src/error.ts';\nimport type {Expand} from '../../../shared/src/expand.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../zero-protocol/src/error-origin.ts';\nimport {ErrorReason} from '../../../zero-protocol/src/error-reason.ts';\nimport type {ProtocolError} from '../../../zero-protocol/src/error.ts';\nimport {\n  type BackoffBody,\n  type ErrorBody,\n  isProtocolError,\n  type PushFailedBody,\n  type TransformFailedBody,\n} from '../../../zero-protocol/src/error.ts';\nimport {ClientErrorKind} from './client-error-kind.ts';\nimport {ConnectionStatus} from './connection-status.ts';\n\nexport type AuthError = ProtocolError<NeedsAuthReason>;\nexport type ClientErrorBody = {\n  kind: ClientErrorKind;\n  origin: typeof ErrorOrigin.Client;\n  message: string;\n};\nexport type ClosedError = ClientError<{\n  kind: ClientErrorKind.ClientClosed;\n  message: string;\n}>;\nexport type NeedsAuthReason = Expand<\n  | (ErrorBody & {\n      kind: ErrorKind.AuthInvalidated | ErrorKind.Unauthorized;\n    })\n  | (Extract<PushFailedBody, {reason: ErrorReason.HTTP}> & {status: 401 | 403})\n  | (Extract<TransformFailedBody, {reason: ErrorReason.HTTP}> & {\n      status: 401 | 403;\n    })\n>;\nexport type OfflineError = ClientError<{\n  kind: ClientErrorKind.Offline;\n  message: string;\n}>;\nexport type NoSocketOriginError = ClientError<{\n  kind: ClientErrorKind.NoSocketOrigin;\n  message: string;\n}>;\nexport type DisconnectedReason = OfflineError | NoSocketOriginError;\nexport type ServerError = ProtocolError<ErrorBody>;\nexport type ZeroError = ServerError | ClientError;\nexport type ZeroErrorBody = Expand<ErrorBody | ClientErrorBody>;\nexport type ZeroErrorDetails = Expand<Omit<ZeroErrorBody, 'message'>>;\nexport type ZeroErrorKind = Expand<ErrorKind | ClientErrorKind>;\n\n/**\n * Represents an error encountered by the Zero client.\n */\nexport class ClientError<\n  const T extends Omit<ClientErrorBody, 'origin'> = Omit<\n    ClientErrorBody,\n    'origin'\n  >,\n> extends Error {\n  readonly errorBody: {origin: typeof ErrorOrigin.Client} & T;\n\n  constructor(errorBody: T, options?: ErrorOptions) {\n    super(errorBody.message, options);\n    this.name = 'ClientError';\n    this.errorBody = {...errorBody, origin: ErrorOrigin.Client};\n  }\n\n  get kind(): T['kind'] {\n    return this.errorBody.kind;\n  }\n}\n\nexport function isZeroError(ex: unknown): ex is ZeroError {\n  return isClientError(ex) || isServerError(ex);\n}\n\nexport function isClientError(ex: unknown): ex is ClientError<ClientErrorBody> {\n  return (\n    ex instanceof ClientError && ex.errorBody.origin === ErrorOrigin.Client\n  );\n}\n\nexport function isServerError(ex: unknown): ex is ServerError {\n  return (\n    isProtocolError(ex) &&\n    (ex.errorBody.origin === ErrorOrigin.Server ||\n      ex.errorBody.origin === ErrorOrigin.ZeroCache)\n  );\n}\n\nexport function isOfflineError(ex: unknown): ex is OfflineError {\n  return isClientError(ex) && ex.kind === ClientErrorKind.Offline;\n}\n\nexport function isAuthError(ex: unknown): ex is AuthError {\n  if (isServerError(ex)) {\n    if (\n      ex.kind === ErrorKind.AuthInvalidated ||\n      ex.kind === ErrorKind.Unauthorized\n    ) {\n      return true;\n    }\n    if (\n      (ex.errorBody.kind === ErrorKind.PushFailed ||\n        ex.errorBody.kind === ErrorKind.TransformFailed) &&\n      ex.errorBody.reason === ErrorReason.HTTP &&\n      (ex.errorBody.status === 401 || ex.errorBody.status === 403)\n    ) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nexport function getBackoffParams(error: ZeroError): BackoffBody | undefined {\n  if (isServerError(error)) {\n    switch (error.errorBody.kind) {\n      case ErrorKind.Rebalance:\n      case ErrorKind.Rehome:\n      case ErrorKind.ServerOverloaded:\n        return error.errorBody;\n    }\n  }\n  return undefined;\n}\n\nexport const NO_STATUS_TRANSITION = 'NO_STATUS_TRANSITION';\n\nexport type ErrorConnectionTransition =\n  | {status: typeof NO_STATUS_TRANSITION; reason: ZeroError}\n  | {status: ConnectionStatus.NeedsAuth; reason: AuthError}\n  | {status: ConnectionStatus.Error; reason: ZeroError}\n  | {status: ConnectionStatus.Disconnected; reason: DisconnectedReason}\n  | {status: ConnectionStatus.Closed; reason: ZeroError};\n\n/**\n * Returns the status to transition to, or null if the error\n * indicates that the connection should continue in the current state.\n */\nexport function getErrorConnectionTransition(\n  ex: unknown,\n): ErrorConnectionTransition {\n  // Handle auth errors by transitioning to needs-auth state\n  if (isAuthError(ex)) {\n    return {\n      status: ConnectionStatus.NeedsAuth,\n      reason: ex,\n    } as const;\n  }\n\n  if (isClientError(ex)) {\n    switch (ex.kind) {\n      // Connecting errors that should continue in the current state\n      case ClientErrorKind.AbruptClose:\n      case ClientErrorKind.CleanClose:\n      case ClientErrorKind.ConnectTimeout:\n      case ClientErrorKind.PingTimeout:\n      case ClientErrorKind.PullTimeout:\n      case ClientErrorKind.Hidden:\n      case ClientErrorKind.UnexpectedBaseCookie:\n        return {status: NO_STATUS_TRANSITION, reason: ex} as const;\n\n      // Fatal errors that should transition to error state\n      case ClientErrorKind.Internal:\n      case ClientErrorKind.InvalidMessage:\n      case ClientErrorKind.UserDisconnect:\n        return {status: ConnectionStatus.Error, reason: ex} as const;\n\n      // Disconnected error (this should already result in a disconnected state)\n      case ClientErrorKind.Offline:\n      case ClientErrorKind.NoSocketOrigin:\n        return {\n          status: ConnectionStatus.Disconnected,\n          reason: ex as DisconnectedReason,\n        } as const;\n\n      // Closed error (this should already result in a closed state)\n      case ClientErrorKind.ClientClosed:\n        return {status: ConnectionStatus.Closed, reason: ex} as const;\n\n      default:\n        unreachable(ex.kind);\n    }\n  }\n\n  if (isServerError(ex)) {\n    switch (ex.kind) {\n      // Errors that should transition to error state\n      case ErrorKind.ClientNotFound:\n      case ErrorKind.InvalidConnectionRequest:\n      case ErrorKind.InvalidConnectionRequestBaseCookie:\n      case ErrorKind.InvalidConnectionRequestLastMutationID:\n      case ErrorKind.InvalidConnectionRequestClientDeleted:\n      case ErrorKind.InvalidMessage:\n      case ErrorKind.InvalidPush:\n      case ErrorKind.VersionNotSupported:\n      case ErrorKind.SchemaVersionNotSupported:\n      case ErrorKind.Internal:\n      // PushFailed and TransformFailed can be auth errors (401/403)\n      // or other errors - handle non-auth cases here\n      case ErrorKind.PushFailed:\n      case ErrorKind.TransformFailed:\n        return {status: ConnectionStatus.Error, reason: ex} as const;\n\n      // Errors that should continue with backoff/retry\n      case ErrorKind.Rebalance:\n      case ErrorKind.Rehome:\n      case ErrorKind.ServerOverloaded:\n        return {status: NO_STATUS_TRANSITION, reason: ex} as const;\n\n      // Auth errors are handled above by isAuthError check\n      case ErrorKind.AuthInvalidated:\n      case ErrorKind.Unauthorized:\n        return {\n          status: ConnectionStatus.NeedsAuth,\n          reason: ex as AuthError,\n        } as const;\n\n      // Mutation-specific errors don't affect connection state\n      case ErrorKind.MutationRateLimited:\n      case ErrorKind.MutationFailed:\n        return {status: NO_STATUS_TRANSITION, reason: ex} as const;\n\n      default:\n        unreachable(ex.kind);\n    }\n  }\n\n  // we default to error state if we don't know what to do\n  // this is a catch-all for unexpected errors\n  return {\n    status: ConnectionStatus.Error,\n    reason: new ClientError(\n      {\n        kind: ClientErrorKind.Internal,\n        message: 'Unexpected internal error: ' + getErrorMessage(ex),\n      },\n      {cause: ex},\n    ),\n  } as const;\n}\n"],"names":["ErrorOrigin.Client","ErrorOrigin.Server","ErrorOrigin.ZeroCache","ErrorKind.AuthInvalidated","ErrorKind.Unauthorized","ErrorKind.PushFailed","ErrorKind.TransformFailed","ErrorReason.HTTP","ErrorKind.Rebalance","ErrorKind.Rehome","ErrorKind.ServerOverloaded","ConnectionStatus.NeedsAuth","ClientErrorKind.AbruptClose","ClientErrorKind.CleanClose","ClientErrorKind.ConnectTimeout","ClientErrorKind.PingTimeout","ClientErrorKind.PullTimeout","ClientErrorKind.Hidden","ClientErrorKind.UnexpectedBaseCookie","ClientErrorKind.Internal","ClientErrorKind.InvalidMessage","ClientErrorKind.UserDisconnect","ConnectionStatus.Error","ClientErrorKind.Offline","ClientErrorKind.NoSocketOrigin","ConnectionStatus.Disconnected","ClientErrorKind.ClientClosed","ConnectionStatus.Closed","ErrorKind.ClientNotFound","ErrorKind.InvalidConnectionRequest","ErrorKind.InvalidConnectionRequestBaseCookie","ErrorKind.InvalidConnectionRequestLastMutationID","ErrorKind.InvalidConnectionRequestClientDeleted","ErrorKind.InvalidMessage","ErrorKind.InvalidPush","ErrorKind.VersionNotSupported","ErrorKind.SchemaVersionNotSupported","ErrorKind.Internal","ErrorKind.MutationRateLimited","ErrorKind.MutationFailed"],"mappings":";;;;;;;;AAsDO,MAAM,oBAKH,MAAM;AAAA,EACL;AAAA,EAET,YAAY,WAAc,SAAwB;AAChD,UAAM,UAAU,SAAS,OAAO;AAChC,SAAK,OAAO;AACZ,SAAK,YAAY,EAAC,GAAG,WAAW,QAAQA,OAAY;AAAA,EACtD;AAAA,EAEA,IAAI,OAAkB;AACpB,WAAO,KAAK,UAAU;AAAA,EACxB;AACF;AAEO,SAAS,YAAY,IAA8B;AACxD,SAAO,cAAc,EAAE,KAAK,cAAc,EAAE;AAC9C;AAEO,SAAS,cAAc,IAAiD;AAC7E,SACE,cAAc,eAAe,GAAG,UAAU,WAAWA;AAEzD;AAEO,SAAS,cAAc,IAAgC;AAC5D,SACE,gBAAgB,EAAE,MACjB,GAAG,UAAU,WAAWC,UACvB,GAAG,UAAU,WAAWC;AAE9B;AAMO,SAAS,YAAY,IAA8B;AACxD,MAAI,cAAc,EAAE,GAAG;AACrB,QACE,GAAG,SAASC,mBACZ,GAAG,SAASC,cACZ;AACA,aAAO;AAAA,IACT;AACA,SACG,GAAG,UAAU,SAASC,cACrB,GAAG,UAAU,SAASC,oBACxB,GAAG,UAAU,WAAWC,SACvB,GAAG,UAAU,WAAW,OAAO,GAAG,UAAU,WAAW,MACxD;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAEO,SAAS,iBAAiB,OAA2C;AAC1E,MAAI,cAAc,KAAK,GAAG;AACxB,YAAQ,MAAM,UAAU,MAAA;AAAA,MACtB,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AACH,eAAO,MAAM;AAAA,IAAA;AAAA,EAEnB;AACA,SAAO;AACT;AAEO,MAAM,uBAAuB;AAa7B,SAAS,6BACd,IAC2B;AAE3B,MAAI,YAAY,EAAE,GAAG;AACnB,WAAO;AAAA,MACL,QAAQC;AAAAA,MACR,QAAQ;AAAA,IAAA;AAAA,EAEZ;AAEA,MAAI,cAAc,EAAE,GAAG;AACrB,YAAQ,GAAG,MAAA;AAAA;AAAA,MAET,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AACH,eAAO,EAAC,QAAQ,sBAAsB,QAAQ,GAAA;AAAA;AAAA,MAGhD,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AACH,eAAO,EAAC,QAAQC,SAAwB,QAAQ,GAAA;AAAA;AAAA,MAGlD,KAAKC;AAAAA,MACL,KAAKC;AACH,eAAO;AAAA,UACL,QAAQC;AAAAA,UACR,QAAQ;AAAA,QAAA;AAAA;AAAA,MAIZ,KAAKC;AACH,eAAO,EAAC,QAAQC,QAAyB,QAAQ,GAAA;AAAA,MAEnD;AACE,oBAAY,GAAG,IAAI;AAAA,IAAA;AAAA,EAEzB;AAEA,MAAI,cAAc,EAAE,GAAG;AACrB,YAAQ,GAAG,MAAA;AAAA;AAAA,MAET,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AAAAA;AAAAA;AAAAA,MAGL,KAAKhC;AAAAA,MACL,KAAKC;AACH,eAAO,EAAC,QAAQgB,SAAwB,QAAQ,GAAA;AAAA;AAAA,MAGlD,KAAKd;AAAAA,MACL,KAAKC;AAAAA,MACL,KAAKC;AACH,eAAO,EAAC,QAAQ,sBAAsB,QAAQ,GAAA;AAAA;AAAA,MAGhD,KAAKP;AAAAA,MACL,KAAKC;AACH,eAAO;AAAA,UACL,QAAQO;AAAAA,UACR,QAAQ;AAAA,QAAA;AAAA;AAAA,MAIZ,KAAK2B;AAAAA,MACL,KAAKC;AACH,eAAO,EAAC,QAAQ,sBAAsB,QAAQ,GAAA;AAAA,MAEhD;AACE,oBAAY,GAAG,IAAI;AAAA,IAAA;AAAA,EAEzB;AAIA,SAAO;AAAA,IACL,QAAQjB;AAAAA,IACR,QAAQ,IAAI;AAAA,MACV;AAAA,QACE,MAAMH;AAAAA,QACN,SAAS,gCAAgC,gBAAgB,EAAE;AAAA,MAAA;AAAA,MAE7D,EAAC,OAAO,GAAA;AAAA,IAAE;AAAA,EACZ;AAEJ;"}

package/out/zero-client/src/client/version.js

@@ -1,4 +1,4 @@
-const version = "0.25.9-canary.2";
+const version = "0.25.9";
 export {
   version
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rocicorp/zero",
-  "version": "0.25.9-canary.2",
+  "version": "0.25.9",
   "description": "Zero is a web framework for serverless web development.",
   "author": "Rocicorp, Inc.",
   "repository": {