@rocicorp/zero 1.12.0-canary.12 → 1.12.0-canary.13

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":["v"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {PreSerialized, Sized, Source} from '../../types/streams.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n  changeStreamDataSchema,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {LocalReplica, SubscribeDownstream} from './subscribe.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering while replaying stored changes and\n * filtering changes already seen by a subscriber.\n *\n * Only commit watermarks are exposed to subscribers. The JSON string is the\n * canonical serialization shared by storage and live forwarding.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date. Each result contains parsed, validated data and its approximate\n   * serialized transport size.\n   */\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n//   - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n//     the upstream commit timestamp for end-to-end serving lag measurement.\n//     The stream is parsed in 'passthrough' mode, so an older peer ignores the\n//     field, and a newer peer treats its absence (including in changes\n//     replayed from the Change DB) as \"no commit time reported\".\n// v7: v1.12\n//   - Merges the separate `/snapshot` and `/changes` connections into a single\n//     bidirectional `/subscribe` connection (see subscribe.ts). The view-syncer\n//     drives the connection with upstream `reserve-snapshot` / `start-subscription`\n//     application messages; the change-streamer replies with a `reserved`\n//     message and then the change stream on the same socket. This guarantees the\n//     replication-manager that reserves the change log is the one that serves the\n//     subscription. The legacy `/snapshot` + `/changes` endpoints remain for\n//     v4-v6 subscribers during rollout.\n//   - Drops the legacy \"initial\" and \"logsChangeStream\" query parameters from the\n//     subscriber context\n// v8:\n//   - Adds `backfills` to the `start-subscription` context, with which the\n//     subscriber reports its pending backfills (and their progress).\n//   - Adds `progressMarks` to `backfill` and `backfill-completed` messages,\n//     which subscribers use to ignore backfill data that would otherwise\n//     create a gap (see `acceptBackfill()`).\n//   Both fields are optional in the schemas so that v7 peers (and changes\n//   persisted in the change-log by v7 replication-managers) remain compatible,\n//   but they should be considered non-optional as of v8.\n\nexport const PROTOCOL_VERSION = 8;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string;\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether the subscriber supports batched WebSocket frames.\n   */\n  wsBatched?: boolean | undefined;\n\n  /**\n   * The subscriber's pending backfills as of its `watermark`. Reported by\n   * subscribers as of protocol v8. `undefined` for older subscribers.\n   */\n  backfills?: BackfillRequest[] | undefined;\n\n  /**\n   * Whether the subscriber understands the `draining` status, i.e. it will\n   * switch over to another replication-manager when the current one shuts\n   * down. A draining replication-manager waits for drainable subscribers to\n   * leave before exiting.\n   */\n  drainable?: boolean | undefined;\n\n  /**\n   * Whether the subscriber is switching over from another subscription with\n   * a replica that it is currently serving. If the replica version is\n   * incompatible, such a subscriber is held (see {@link Status.restore})\n   * rather than being immediately rejected. Subscribers that are booting up\n   * never set this.\n   */\n  switchover?: boolean | undefined;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nconst statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n\n  /**\n   * Sent by a replication-manager that is shutting down to its (drainable)\n   * serving subscribers. The subscriber should switch over to another\n   * replication-manager (see {@link SubscriberContext.switchover}) while\n   * continuing to consume this subscription until the switchover succeeds.\n   */\n  draining: v.boolean().optional(),\n\n  /**\n   * Sent on a {@link SubscriberContext.switchover switchover} subscription\n   * to a replication-manager running an incompatible replica version.\n   * Rather than being immediately rejected with\n   * {@link ErrorType.WrongReplicaVersion}, the subscriber is held until it\n   * is granted permission to restore the replication-manager's backup,\n   * which limits the number of subscribers that are restoring (and thus not\n   * serving) at the same time.\n   *\n   * * `queued`: the subscriber is waiting for permission to restore. It\n   *   should continue consuming its existing subscription (if any).\n   * * `granted`: the subscriber should (gracefully) restart to restore a\n   *   new replica.\n   */\n  restore: v\n    .union(\n      v.object({state: v.literal('queued'), position: v.number()}),\n      v.object({state: v.literal('granted')}),\n    )\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nconst statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n  /**\n   * The replica version of the replication-manager, sent with\n   * {@link ErrorType.WrongReplicaVersion} so that a subscriber with a newer\n   * replica can avoid restoring an older one.\n   */\n  replicaVersion: v.string().optional(),\n});\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\n/** A downstream message and its approximate serialized transport size. */\nexport type SizedDownstream = Sized<Downstream>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads or pre-serialized Buffers.\n   *\n   * If a `downstream` subscription is supplied it is used directly as the\n   * returned Source. Otherwise, a new one is created.\n   */\n  subscribe(\n    ctx: SubscriberContext,\n    downstream?: Subscription<string | PreSerializedBatch>,\n  ): Promise<Source<string | PreSerialized>>;\n\n  /**\n   * Starts a snapshot reservation to preserve change-log entries while\n   * a soon-to-be-subscriber downloads the current replica backup. Once\n   * the change-streamer knows that a backup is available\n   * (i.e. from its backup source), it sends a confirmation\n   * SnapshotMessage and reserves the corresponding entries in the change-log\n   * until the client with that `taskID` subscribes, or if the snapshot\n   * connection terminates.\n   */\n  startSnapshotReservation(\n    taskID: string,\n    opts?: {localReplica?: LocalReplica | undefined},\n  ): Promise<Source<SnapshotMessage>>;\n\n  /**\n   * Signals (drainable) serving subscribers to switch over to another\n   * replication-manager, and resolves when they have all disconnected.\n   * Subscribers continue to be served in the meantime.\n   */\n  drain?(): Promise<void>;\n}\n"],"mappings":";;;;;;;;AA2MA,IAAM,eAAeA,eAAE,OAAO;CAC5B,KAAKA,eAAE,QAAQ,QAAQ;CAEvB,WAAWA,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgBA,eAAE,OAAO;CAC3B,CAAC,CAAC,CACD,SAAS;;;;;;;CAQZ,UAAUA,eAAE,QAAQ,CAAC,CAAC,SAAS;;;;;;;;;;;;;;;CAgB/B,SAASA,eACN,MACCA,eAAE,OAAO;EAAC,OAAOA,eAAE,QAAQ,QAAQ;EAAG,UAAUA,eAAE,OAAO;CAAC,CAAC,GAC3DA,eAAE,OAAO,EAAC,OAAOA,eAAE,QAAQ,SAAS,EAAC,CAAC,CACxC,CAAC,CACA,SAAS;AACd,CAAC;AAID,IAAM,sBAAsBA,eAAE,MAAM,CAACA,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAEvE,IAAM,0BAA0BA,eAAE,OAAO;CACvC,MAAMA,eAAE,OAAO;CACf,SAASA,eAAE,OAAO,CAAC,CAAC,SAAS;;;;;;CAM7B,gBAAgBA,eAAE,OAAO,CAAC,CAAC,SAAS;AACtC,CAAC;AAED,IAAM,cAAcA,eAAE,MAAM,CAACA,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmBA,eAAE,MAChC,qBACA,wBACA,WACF;AAEA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}
+{"version":3,"file":"change-streamer.js","names":["v"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {PreSerialized, Sized, Source} from '../../types/streams.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n  changeStreamDataSchema,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {LocalReplica, SubscribeDownstream} from './subscribe.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering while replaying stored changes and\n * filtering changes already seen by a subscriber.\n *\n * Only commit watermarks are exposed to subscribers. The JSON string is the\n * canonical serialization shared by storage and live forwarding.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date. Each result contains parsed, validated data and its approximate\n   * serialized transport size.\n   */\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n//   - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n//     the upstream commit timestamp for end-to-end serving lag measurement.\n//     The stream is parsed in 'passthrough' mode, so an older peer ignores the\n//     field, and a newer peer treats its absence (including in changes\n//     replayed from the Change DB) as \"no commit time reported\".\n// v7: v1.12\n//   - Merges the separate `/snapshot` and `/changes` connections into a single\n//     bidirectional `/subscribe` connection (see subscribe.ts). The view-syncer\n//     drives the connection with upstream `reserve-snapshot` / `start-subscription`\n//     application messages; the change-streamer replies with a `reserved`\n//     message and then the change stream on the same socket. This guarantees the\n//     replication-manager that reserves the change log is the one that serves the\n//     subscription. The legacy `/snapshot` + `/changes` endpoints remain for\n//     v4-v6 subscribers during rollout.\n//   - Drops the legacy \"initial\" and \"logsChangeStream\" query parameters from the\n//     subscriber context\n// v8:\n//   - Adds `backfills` to the `start-subscription` context, with which the\n//     subscriber reports its pending backfills (and their progress).\n//   - Adds `progressMarks` to `backfill` and `backfill-completed` messages,\n//     which subscribers use to ignore backfill data that would otherwise\n//     create a gap (see `acceptBackfill()`).\n//   Both fields are optional in the schemas so that v7 peers (and changes\n//   persisted in the change-log by v7 replication-managers) remain compatible,\n//   but they should be considered non-optional as of v8.\n\nexport const PROTOCOL_VERSION = 8;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string;\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether the subscriber supports batched WebSocket frames.\n   */\n  wsBatched?: boolean | undefined;\n\n  /**\n   * The subscriber's pending backfills as of its `watermark`. Reported by\n   * subscribers as of protocol v8. `undefined` for older subscribers.\n   */\n  backfills?: BackfillRequest[] | undefined;\n\n  /**\n   * Whether the subscriber understands the `draining` status, i.e. it will\n   * switch over to another replication-manager when the current one shuts\n   * down. A draining replication-manager waits for drainable subscribers to\n   * leave before exiting.\n   */\n  drainable?: boolean | undefined;\n\n  /**\n   * Whether the subscriber is switching over from another subscription with\n   * a replica that it is currently serving. If the replica version is\n   * incompatible, such a subscriber is held (see {@link Status.restore})\n   * rather than being immediately rejected. Subscribers that are booting up\n   * never set this.\n   */\n  switchover?: boolean | undefined;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nconst statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n\n  /**\n   * Sent by a replication-manager that is shutting down to its (drainable)\n   * serving subscribers. The subscriber should switch over to another\n   * replication-manager (see {@link SubscriberContext.switchover}) while\n   * continuing to consume this subscription until the switchover succeeds.\n   */\n  draining: v.boolean().optional(),\n\n  /**\n   * Sent on a {@link SubscriberContext.switchover switchover} subscription\n   * to a replication-manager running an incompatible replica version.\n   * Rather than being immediately rejected with\n   * {@link ErrorType.WrongReplicaVersion}, the subscriber is held until it\n   * is granted permission to restore the replication-manager's backup,\n   * which limits the number of subscribers that are restoring (and thus not\n   * serving) at the same time.\n   *\n   * * `queued`: the subscriber is waiting for permission to restore. It\n   *   should continue consuming its existing subscription (if any).\n   * * `granted`: the subscriber should (gracefully) restart to restore a\n   *   new replica.\n   */\n  restore: v\n    .union(\n      v.object({state: v.literal('queued'), position: v.number()}),\n      v.object({state: v.literal('granted')}),\n    )\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nconst statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n  /**\n   * Sent with {@link ErrorType.WrongReplicaVersion}: `true` when the\n   * replication-manager's replica is older than the subscriber's, so that a\n   * subscriber with a newer replica can avoid restoring an older one.\n   */\n  olderReplica: v.boolean().optional(),\n  /**\n   * @deprecated The replica version of the replication-manager, sent by\n   * older replication-managers in place of `olderReplica`. New\n   * replication-managers do not disclose their replica version to a rejected\n   * subscriber.\n   */\n  replicaVersion: v.string().optional(),\n});\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\n/** A downstream message and its approximate serialized transport size. */\nexport type SizedDownstream = Sized<Downstream>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads or pre-serialized Buffers.\n   *\n   * If a `downstream` subscription is supplied it is used directly as the\n   * returned Source. Otherwise, a new one is created.\n   */\n  subscribe(\n    ctx: SubscriberContext,\n    downstream?: Subscription<string | PreSerializedBatch>,\n  ): Promise<Source<string | PreSerialized>>;\n\n  /**\n   * Starts a snapshot reservation to preserve change-log entries while\n   * a soon-to-be-subscriber downloads the current replica backup. Once\n   * the change-streamer knows that a backup is available\n   * (i.e. from its backup source), it sends a confirmation\n   * SnapshotMessage and reserves the corresponding entries in the change-log\n   * until the client with that `taskID` subscribes, or if the snapshot\n   * connection terminates.\n   */\n  startSnapshotReservation(\n    taskID: string,\n    opts?: {localReplica?: LocalReplica | undefined},\n  ): Promise<Source<SnapshotMessage>>;\n\n  /**\n   * Signals (drainable) serving subscribers to switch over to another\n   * replication-manager, and resolves when they have all disconnected.\n   * Subscribers continue to be served in the meantime.\n   */\n  drain?(): Promise<void>;\n}\n"],"mappings":";;;;;;;;AA2MA,IAAM,eAAeA,eAAE,OAAO;CAC5B,KAAKA,eAAE,QAAQ,QAAQ;CAEvB,WAAWA,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgBA,eAAE,OAAO;CAC3B,CAAC,CAAC,CACD,SAAS;;;;;;;CAQZ,UAAUA,eAAE,QAAQ,CAAC,CAAC,SAAS;;;;;;;;;;;;;;;CAgB/B,SAASA,eACN,MACCA,eAAE,OAAO;EAAC,OAAOA,eAAE,QAAQ,QAAQ;EAAG,UAAUA,eAAE,OAAO;CAAC,CAAC,GAC3DA,eAAE,OAAO,EAAC,OAAOA,eAAE,QAAQ,SAAS,EAAC,CAAC,CACxC,CAAC,CACA,SAAS;AACd,CAAC;AAID,IAAM,sBAAsBA,eAAE,MAAM,CAACA,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAEvE,IAAM,0BAA0BA,eAAE,OAAO;CACvC,MAAMA,eAAE,OAAO;CACf,SAASA,eAAE,OAAO,CAAC,CAAC,SAAS;;;;;;CAM7B,cAAcA,eAAE,QAAQ,CAAC,CAAC,SAAS;;;;;;;CAOnC,gBAAgBA,eAAE,OAAO,CAAC,CAAC,SAAS;AACtC,CAAC;AAED,IAAM,cAAcA,eAAE,MAAM,CAACA,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmBA,eAAE,MAChC,qBACA,wBACA,WACF;AAEA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}

package/out/zero-cache/src/services/change-streamer/subscriber.js.map

@@ -1 +1 @@
-{"version":3,"file":"subscriber.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {resolver, type Resolver} from '../../../../shared/src/resolver.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n  wsBatched?: boolean | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n\n  /**\n   * The subscriber's pending backfills (protocol v8+), which are tracked\n   * over the changes sent to the subscriber.\n   */\n  backfills?: BackfillState | undefined;\n\n  /**\n   * Called once the subscriber has been sent all changes up to the head of\n   * the stream, i.e. its catchup and backlog have been flushed. From this\n   * point on, the subscriber's tracked {@link backfills} reflect the same\n   * position in the stream as that of the change-streamer.\n   */\n  onAligned?: ((subscriber: Subscriber) => void) | undefined;\n\n  /**\n   * Called when a `backfill` or `backfill-completed` message is ignored for\n   * some of the subscriber's pending backfill columns, after the subscriber\n   * is aligned.\n   */\n  onBackfillIgnored?: ((columns: string[]) => void) | undefined;\n\n  /**\n   * Resolves, once it is known whether lag reporting is enabled upstream, to\n   * the means of requesting a lag report, or to `undefined` if it is disabled.\n   *\n   * Subscribers use lag reports to determine that they are caught up (e.g. to\n   * advertise readiness to serve traffic). Once the subscriber is aligned, a\n   * report is therefore requested, so that it is received without waiting out\n   * the report interval. If lag reporting is disabled, a\n   * {@link LAG_REPORTING_DISABLED} status is sent as a stand-in instead.\n   */\n  lagReports?: Promise<LagReports | undefined> | undefined;\n\n  /**\n   * Whether the subscriber switches over to another replication-manager\n   * when this one drains (see `DrainTracker`).\n   */\n  drainable?: boolean | undefined;\n};\n\n/**\n * A status message queued in the backlog, so that it is sent downstream in\n * the order in which it was received relative to (backlogged) changes.\n */\ntype QueuedStatus = {readonly status: string};\n\ntype BacklogEntry = WatermarkedChange | QueuedStatus;\n\nfunction isQueuedStatus(entry: BacklogEntry): entry is QueuedStatus {\n  return !Array.isArray(entry);\n}\n\nfunction entryBytes(entry: BacklogEntry): number {\n  return isQueuedStatus(entry) ? entry.status.length : entry[2].length;\n}\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly drainable: boolean;\n  readonly #downstream: Subscription<string | PreSerializedBatch>;\n  readonly #wsBatched: boolean;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<BacklogEntry> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n  readonly #backfills: BackfillState | undefined;\n  readonly #onAligned: ((subscriber: Subscriber) => void) | undefined;\n  readonly #onBackfillIgnored: ((columns: string[]) => void) | undefined;\n  readonly #lagReports: Promise<LagReports | undefined> | undefined;\n  #aligned = false;\n  // The watermark to be acked for the subscriber to be caught up, set when\n  // it is aligned.\n  #caughtUpWatermark: string | undefined;\n  readonly #caughtUp = resolver<void>();\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string | PreSerializedBatch>,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.drainable = options.drainable ?? false;\n    this.#downstream = downstream;\n    this.#wsBatched = options.wsBatched ?? false;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n    this.#backfills = options.backfills;\n    this.#onAligned = options.onAligned;\n    this.#onBackfillIgnored = options.onBackfillIgnored;\n    this.#lagReports = options.lagReports;\n  }\n\n  /**\n   * The subscriber's tracked pending backfills, or `undefined` if the\n   * subscriber does not report them (i.e. protocol < v8).\n   */\n  get backfills(): BackfillState | undefined {\n    return this.#backfills;\n  }\n\n  /**\n   * Whether the subscriber has been sent all changes up to the head of the\n   * stream (see {@link SubscriberOptions.onAligned}).\n   */\n  get aligned(): boolean {\n    return this.#aligned;\n  }\n\n  /**\n   * Resolves once the subscriber has caught up, i.e. once it has acked\n   * (durably applied) all changes up to the head of the stream at the time\n   * it was {@link aligned}. Never resolves if the subscriber is closed first.\n   */\n  get caughtUp(): Promise<void> {\n    return this.#caughtUp.promise;\n  }\n\n  #trackBackfills(change: WatermarkedChange) {\n    if (this.#backfills) {\n      const ignored = this.#backfills.applySerialized(change);\n      if (ignored.length && this.#aligned) {\n        this.#onBackfillIgnored?.(ignored);\n      }\n    }\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  sendBatch(\n    changes: readonly WatermarkedChange[],\n    preSerialized?: PreSerializedBatch | undefined,\n  ): Promise<void> {\n    if (changes.length === 0) {\n      return promiseVoid;\n    }\n\n    if (\n      this.#wsBatched &&\n      preSerialized &&\n      !this.#backlog &&\n      this.#initialized &&\n      changes[0][0] > this.#watermark &&\n      (this.#protocolVersion >= 5 ||\n        changes.every(c => this.supportsMessage(c[1])))\n    ) {\n      let commitWatermark: string | undefined;\n      for (let i = changes.length - 1; i >= 0; i--) {\n        if (changes[i][1] === 'commit') {\n          commitWatermark = changes[i][0];\n          break;\n        }\n      }\n      if (commitWatermark) {\n        this.#watermark = commitWatermark;\n      }\n      for (const change of changes) {\n        this.#trackBackfills(change);\n      }\n      return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n    }\n\n    const promises: Promise<void>[] = [];\n    for (const change of changes) {\n      const p = this.send(change);\n      if (p !== promiseVoid) {\n        promises.push(p);\n      }\n    }\n    if (promises.length === 0) {\n      return promiseVoid;\n    }\n    if (promises.length === 1) {\n      return promises[0];\n    }\n    return Promise.all(promises).then(() => {});\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      // The initial status precedes the catchup, so it bypasses the backlog.\n      void this.#sendDownstream(['status', {tag: 'status'}]);\n    }\n  }\n\n  /**\n   * Sends a status message (e.g. a lag report) downstream. While the\n   * subscriber is catching up, the status is queued in the backlog behind\n   * the changes that preceded it in the replication stream, so that the\n   * subscriber processes it in stream order. This is necessary for lag\n   * reports to measure the lag of the subscriber, including its catchup.\n   *\n   * Status messages received before the subscriber is initialized are\n   * dropped.\n   */\n  sendStatus(status: Status) {\n    if (!this.#initialized) {\n      return;\n    }\n    const downstream: Downstream = ['status', status];\n    if (this.#backlog) {\n      this.#pushBacklog({status: BigIntJSON.stringify(downstream)});\n      return;\n    }\n    void this.#sendDownstream(downstream);\n  }\n\n  /**\n   * Sends a control status (e.g. `draining` or `restore`) downstream\n   * immediately, regardless of whether the subscriber is initialized or\n   * catching up. Unlike {@link sendStatus}, such statuses are not tied to a\n   * position in the replication stream.\n   */\n  sendControlStatus(status: Status) {\n    void this.#sendDownstream(['status', status]);\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    this.#trackBackfills(change);\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      this.#advanceAck(watermark);\n    }\n  }\n\n  #advanceAck(watermark: string) {\n    // Sends can complete out of order (e.g. the bounded window in\n    // #drainBacklog), so the ack only advances monotonically, and listeners\n    // are only notified when it does.\n    const acked = max(this.#acked, watermark);\n    if (acked !== this.#acked) {\n      this.#acked = acked;\n      this.#onAck?.(acked);\n      this.#checkCaughtUp();\n    }\n  }\n\n  #checkCaughtUp() {\n    if (\n      this.#caughtUpWatermark !== undefined &&\n      this.#acked >= this.#caughtUpWatermark\n    ) {\n      this.#caughtUpWatermark = undefined;\n      this.#caughtUp.resolve();\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  async #sendPreSerializedDownstream(\n    batch: PreSerializedBatch,\n    commitWatermark?: string | undefined,\n  ): Promise<void> {\n    const size = batch.byteLength;\n    this.#pending += batch.changes.length;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(batch);\n    try {\n      const outcome = await result;\n      if (commitWatermark && outcome === 'consumed') {\n        this.#advanceAck(commitWatermark);\n      }\n    } finally {\n      this.#pending -= batch.changes.length;\n      this.#pendingBytes -= size;\n      this.#processed += batch.changes.length;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(\n    error?: ErrorType,\n    message?: string,\n    details?: {replicaVersion?: string | undefined},\n  ) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream([\n        'error',\n        {type: error, message, ...details},\n      ]).finally(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(entry: BacklogEntry) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(entry);\n    this.#backlogBytes += entryBytes(entry);\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const entry = this.#backlog?.shift();\n        if (!entry) {\n          const closed = this.#backlog === null;\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          if (!closed && !this.#aligned) {\n            // All changes up to the head have been handed to #sendChange()\n            // (and thus tracked), and subsequent changes are sent directly.\n            this.#aligned = true;\n            this.#onAligned?.(this);\n            this.#caughtUpWatermark = this.#watermark;\n            this.#checkCaughtUp();\n            void this.#lagReports?.then(lagReports => {\n              if (lagReports) {\n                lagReports.request();\n              } else {\n                this.sendStatus(LAG_REPORTING_DISABLED);\n              }\n            });\n          }\n          break;\n        }\n\n        const bytes = entryBytes(entry);\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const send = isQueuedStatus(entry)\n          ? this.#sendStringifiedDownstream(entry.status).then(() => {})\n          : this.#sendChange(entry);\n        const promise = send.finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n\n/** Requests lag reports from upstream (see {@link SubscriberOptions.lagReports}). */\nexport type LagReports = {\n  /**\n   * Requests that a lag report be sent now rather than at the next report\n   * interval. Requests are coalesced upstream, e.g. with a report in flight.\n   */\n  request(): void;\n};\n\n/**\n * Sent to an aligned subscriber when lag reporting is disabled upstream,\n * signaling that the subscriber is caught up. This is a lag report with a\n * `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured\n * (see `isLagReportingDisabledSignal()` in the replicator's recorder).\n */\nexport const LAG_REPORTING_DISABLED: Status = {\n  tag: 'status',\n  lagReport: {nextSendTimeMs: 0},\n};\n"],"mappings":";;;;;;;;AAsBA,IAAM,mCAAmC;AACzC,IAAM,kCAAkC;AA8DxC,SAAS,eAAe,OAA4C;CAClE,OAAO,CAAC,MAAM,QAAQ,KAAK;AAC7B;AAEA,SAAS,WAAW,OAA6B;CAC/C,OAAO,eAAe,KAAK,IAAI,MAAM,OAAO,SAAS,MAAM,EAAE,CAAC;AAChE;;;;;;;;AAuBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CACA;CACA;CACA;CACA;CACA,WAAW;CAGX;CACA,YAAqB,SAAe;CAEpC,YACE,iBACA,IACA,MACA,WACA,YACA,UAA6B,CAAC,GAC9B;EACA,KAAK,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAK,YAAY,QAAQ,aAAa;EACtC,KAAK,cAAc;EACnB,KAAK,aAAa,QAAQ,aAAa;EACvC,KAAK,aAAa;EAClB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,WAAW;EAC/B,KAAK,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAK,SAAS,QAAQ;EACtB,KAAK,aAAa,QAAQ;EAC1B,KAAK,aAAa,QAAQ;EAC1B,KAAK,qBAAqB,QAAQ;EAClC,KAAK,cAAc,QAAQ;CAC7B;;;;;CAMA,IAAI,YAAuC;EACzC,OAAO,KAAK;CACd;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAK;CACd;;;;;;CAOA,IAAI,WAA0B;EAC5B,OAAO,KAAK,UAAU;CACxB;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAK,YAAY;GACnB,MAAM,UAAU,KAAK,WAAW,gBAAgB,MAAM;GACtD,IAAI,QAAQ,UAAU,KAAK,UACzB,KAAK,qBAAqB,OAAO;EAErC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAK;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAK;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAK,yBAAyB,KAAK,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAK,aAAa,QAAQ,KAAK,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAK,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAK,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAK,YAAY;GAC/B,IAAI,KAAK,UAAU;IAIjB,KAAK,aAAa,MAAM;IACxB,OAAO,KAAK,0BAA0B;GACxC;GACA,OAAO,KAAK,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAK,cACL,iBACA,CAAC,KAAK,YACN,KAAK,gBACL,QAAQ,EAAE,CAAC,KAAK,KAAK,eACpB,KAAK,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,EAAE,CAAC,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,EAAE,CAAC;IAC7B;GACF;GAEF,IAAI,iBACF,KAAK,aAAa;GAEpB,KAAK,MAAM,UAAU,SACnB,KAAK,gBAAgB,MAAM;GAE7B,OAAO,KAAK,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAK,cAAc;GACtB,KAAK,eAAe;GAEpB,KAAU,gBAAgB,CAAC,UAAU,EAAC,KAAK,SAAQ,CAAC,CAAC;EACvD;CACF;;;;;;;;;;;CAYA,WAAW,QAAgB;EACzB,IAAI,CAAC,KAAK,cACR;EAEF,MAAM,aAAyB,CAAC,UAAU,MAAM;EAChD,IAAI,KAAK,UAAU;GACjB,KAAK,aAAa,EAAC,QAAQ,WAAW,UAAU,UAAU,EAAC,CAAC;GAC5D;EACF;EACA,KAAU,gBAAgB,UAAU;CACtC;;;;;;;CAQA,kBAAkB,QAAgB;EAChC,KAAU,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAC9C;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAK,YAAY;EACjB,MAAM,KAAK,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAK,YAAY;EACjB,IAAI,CAAC,KAAK,UACR,OAAO,KAAK,iBAAiB;EAE/B,IAAI,CAAC,KAAK,eAAe;GAKvB,KAAK,gBAAgB,KAAK,cAAc;GACxC,KAAU,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAK;CACd;CAEA,MAAM,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAK,aAAa;EAEpB,KAAK,gBAAgB,MAAM;EAC3B,MAAM,SAAS,MAAM,KAAK,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YACjC,KAAK,YAAY,SAAS;CAE9B;CAEA,YAAY,WAAmB;EAI7B,MAAM,QAAQ,IAAI,KAAK,QAAQ,SAAS;EACxC,IAAI,UAAU,KAAK,QAAQ;GACzB,KAAK,SAAS;GACd,KAAK,SAAS,KAAK;GACnB,KAAK,eAAe;EACtB;CACF;CAEA,iBAAiB;EACf,IACE,KAAK,uBAAuB,KAAA,KAC5B,KAAK,UAAU,KAAK,oBACpB;GACA,KAAK,qBAAqB,KAAA;GAC1B,KAAK,UAAU,QAAQ;EACzB;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAK,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAM,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAK;EACL,KAAK,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAK;GACL,KAAK,iBAAiB;GACtB,KAAK;EACP;CACF;CAEA,MAAM,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAK,YAAY,MAAM,QAAQ;EAC/B,KAAK,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YACjC,KAAK,YAAY,eAAe;EAEpC,UAAU;GACR,KAAK,YAAY,MAAM,QAAQ;GAC/B,KAAK,iBAAiB;GACtB,KAAK,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAK,WAAW,KAAK;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAK;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAK,SAAS,UAAU,YAC7B,KAAK,SAAS,MAAM;EAEtB,KAAK,SAAS,KAAK;GAAC,WAAW,KAAK;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAK,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAK;GACd,cAAc,KAAK;GACnB,oBAAoB,KAAK;GACzB,mBAAmB,KAAK;EAC1B;EAEF,MAAM,OAAO,KAAK,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAK,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAK;GACd,cAAc,KAAK;GACnB,oBAAoB,KAAK;GACzB,mBAAmB,KAAK;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAK,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAK,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAK,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAK,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAK,oBAAoB;EACzB,OAAO,MAAM,KAAK;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAK,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MACE,OACA,SACA,SACA;EAGA,KAAK,WAAW;EAChB,KAAK,gBAAgB;EACrB,KAAK,qBAAqB,WAAW;EAGrC,KAAK,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAU,gBAAgB,CACxB,SACA;GAAC,MAAM;GAAO;GAAS,GAAG;EAAO,CACnC,CAAC,CAAC,CAAC,cAAc,KAAK,YAAY,OAAO,CAAC;OAE1C,KAAK,YAAY,OAAO;CAE5B;CAEA,IAAI,gBAAgB;EAClB,OAAO,KAAK,UAAU,QAAQ;CAChC;CAEA,IAAI,wBAAwB;EAI1B,OAAO,KAAK,gBAAgB,KAAK;CACnC;CAEA,IAAI,sBAAsB;EACxB,OAAO,KAAK,wBAAwB,KAAK;CAC3C;CAEA,aAAa,OAAqB;EAChC,OAAO,KAAK,UAAU,gDAAgD;EACtE,KAAK,SAAS,KAAK,KAAK;EACxB,KAAK,iBAAiB,WAAW,KAAK;EACtC,IAAI,KAAK,aACP,KAAK,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAK,mBAAmB;EAC5C,KAAK,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAK,qBAAqB,aAAa,KAAK,qBAAqB;CAC1E;CAEA,MAAM,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,QAAQ,KAAK,UAAU,MAAM;IACnC,IAAI,CAAC,OAAO;KACV,MAAM,SAAS,KAAK,aAAa;KACjC,KAAK,WAAW;KAChB,KAAK,gBAAgB;KACrB,KAAK,qBAAqB,uBACxB,KAAK,qBACP;KACA,IAAI,CAAC,UAAU,CAAC,KAAK,UAAU;MAG7B,KAAK,WAAW;MAChB,KAAK,aAAa,IAAI;MACtB,KAAK,qBAAqB,KAAK;MAC/B,KAAK,eAAe;MACpB,KAAU,aAAa,MAAK,eAAc;OACxC,IAAI,YACF,WAAW,QAAQ;YAEnB,KAAK,WAAW,sBAAsB;MAE1C,CAAC;KACH;KACA;IACF;IAEA,MAAM,QAAQ,WAAW,KAAK;IAC9B,KAAK,iBAAiB;IACtB,KAAK,yBAAyB;IAC9B,KAAK,qBAAqB,uBACxB,KAAK,qBACP;IAQA,MAAM,WAHO,eAAe,KAAK,IAC7B,KAAK,2BAA2B,MAAM,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC,IAC3D,KAAK,YAAY,KAAK,EAAA,CACL,cAAc;KACjC,KAAK,yBAAyB;KAC9B,KAAK,qBAAqB,uBACxB,KAAK,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAK,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAK,gBAAgB;GACrB,KAAK,qBAAqB,uBACxB,KAAK,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAK,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAK,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAK,SAAS,WAAW,KAAK,gBAAgB,KAAK,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAK,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF;;;;;;;AAiBA,IAAa,yBAAiC;CAC5C,KAAK;CACL,WAAW,EAAC,gBAAgB,EAAC;AAC/B"}
+{"version":3,"file":"subscriber.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {resolver, type Resolver} from '../../../../shared/src/resolver.ts';\nimport {RingBuffer} from '../../../../shared/src/ring-buffer.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {\n  ChangeTag,\n  Downstream,\n  Status,\n  WatermarkedChange,\n} from './change-streamer.ts';\nimport type * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst DEFAULT_BACKLOG_HIGH_WATER_BYTES = 16 * 1024 * 1024;\nconst DEFAULT_BACKLOG_LOW_WATER_RATIO = 0.8;\n\nexport type SubscriberOptions = {\n  backlogHighWaterBytes?: number | undefined;\n  backlogLowWaterRatio?: number | undefined;\n  wsBatched?: boolean | undefined;\n\n  /**\n   * Called whenever the subscriber's acked watermark advances, i.e. when it\n   * has confirmed that a commit was durably applied to its replica. The ACK\n   * therefore lags the subscriber's replica rather than leading it.\n   */\n  onAck?: ((watermark: string) => void) | undefined;\n\n  /**\n   * The subscriber's pending backfills (protocol v8+), which are tracked\n   * over the changes sent to the subscriber.\n   */\n  backfills?: BackfillState | undefined;\n\n  /**\n   * Called once the subscriber has been sent all changes up to the head of\n   * the stream, i.e. its catchup and backlog have been flushed. From this\n   * point on, the subscriber's tracked {@link backfills} reflect the same\n   * position in the stream as that of the change-streamer.\n   */\n  onAligned?: ((subscriber: Subscriber) => void) | undefined;\n\n  /**\n   * Called when a `backfill` or `backfill-completed` message is ignored for\n   * some of the subscriber's pending backfill columns, after the subscriber\n   * is aligned.\n   */\n  onBackfillIgnored?: ((columns: string[]) => void) | undefined;\n\n  /**\n   * Resolves, once it is known whether lag reporting is enabled upstream, to\n   * the means of requesting a lag report, or to `undefined` if it is disabled.\n   *\n   * Subscribers use lag reports to determine that they are caught up (e.g. to\n   * advertise readiness to serve traffic). Once the subscriber is aligned, a\n   * report is therefore requested, so that it is received without waiting out\n   * the report interval. If lag reporting is disabled, a\n   * {@link LAG_REPORTING_DISABLED} status is sent as a stand-in instead.\n   */\n  lagReports?: Promise<LagReports | undefined> | undefined;\n\n  /**\n   * Whether the subscriber switches over to another replication-manager\n   * when this one drains (see `DrainTracker`).\n   */\n  drainable?: boolean | undefined;\n};\n\n/**\n * A status message queued in the backlog, so that it is sent downstream in\n * the order in which it was received relative to (backlogged) changes.\n */\ntype QueuedStatus = {readonly status: string};\n\ntype BacklogEntry = WatermarkedChange | QueuedStatus;\n\nfunction isQueuedStatus(entry: BacklogEntry): entry is QueuedStatus {\n  return !Array.isArray(entry);\n}\n\nfunction entryBytes(entry: BacklogEntry): number {\n  return isQueuedStatus(entry) ? entry.status.length : entry[2].length;\n}\n\nexport type BacklogFullWait = {\n  readonly promise: Promise<void>;\n  cancel(): void;\n};\n\nexport type SubscriberStats = {\n  processRate: number;\n  pending: number;\n  backlog: number;\n  backlogBytes: number;\n  totalBufferedBytes: number;\n  missedLastTimeout: boolean;\n};\n\n/**\n * Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n */\nexport class Subscriber {\n  readonly #protocolVersion: number;\n  readonly id: string;\n  readonly mode: ReplicatorMode;\n  readonly drainable: boolean;\n  readonly #downstream: Subscription<string | PreSerializedBatch>;\n  readonly #wsBatched: boolean;\n  #watermark: string;\n  #acked: string;\n  #backlog: RingBuffer<BacklogEntry> | null;\n  // While catchup is running, live changes are buffered here instead of being\n  // pushed downstream. RingBuffer lets drainBacklog consume that backlog without\n  // shifting an array, which matters when a subscriber is far behind.\n  #backlogBytes = 0;\n  #backlogInFlightBytes = 0;\n  #backlogDrain: Promise<void> | null = null;\n  readonly #backlogBackpressure: ByteBackpressureGate;\n  readonly #backlogFullWaiters = new Set<Resolver<void>>();\n  readonly #onAck: ((watermark: string) => void) | undefined;\n  readonly #backfills: BackfillState | undefined;\n  readonly #onAligned: ((subscriber: Subscriber) => void) | undefined;\n  readonly #onBackfillIgnored: ((columns: string[]) => void) | undefined;\n  readonly #lagReports: Promise<LagReports | undefined> | undefined;\n  #aligned = false;\n  // The watermark to be acked for the subscriber to be caught up, set when\n  // it is aligned.\n  #caughtUpWatermark: string | undefined;\n  readonly #caughtUp = resolver<void>();\n\n  constructor(\n    protocolVersion: number,\n    id: string,\n    mode: ReplicatorMode,\n    watermark: string,\n    downstream: Subscription<string | PreSerializedBatch>,\n    options: SubscriberOptions = {},\n  ) {\n    this.#protocolVersion = protocolVersion;\n    this.id = id;\n    this.mode = mode;\n    this.drainable = options.drainable ?? false;\n    this.#downstream = downstream;\n    this.#wsBatched = options.wsBatched ?? false;\n    this.#watermark = watermark;\n    this.#acked = watermark;\n    this.#backlog = new RingBuffer();\n    this.#backlogBackpressure = new ByteBackpressureGate(\n      options.backlogHighWaterBytes ?? DEFAULT_BACKLOG_HIGH_WATER_BYTES,\n      options.backlogLowWaterRatio ?? DEFAULT_BACKLOG_LOW_WATER_RATIO,\n    );\n    this.#onAck = options.onAck;\n    this.#backfills = options.backfills;\n    this.#onAligned = options.onAligned;\n    this.#onBackfillIgnored = options.onBackfillIgnored;\n    this.#lagReports = options.lagReports;\n  }\n\n  /**\n   * The subscriber's tracked pending backfills, or `undefined` if the\n   * subscriber does not report them (i.e. protocol < v8).\n   */\n  get backfills(): BackfillState | undefined {\n    return this.#backfills;\n  }\n\n  /**\n   * Whether the subscriber has been sent all changes up to the head of the\n   * stream (see {@link SubscriberOptions.onAligned}).\n   */\n  get aligned(): boolean {\n    return this.#aligned;\n  }\n\n  /**\n   * Resolves once the subscriber has caught up, i.e. once it has acked\n   * (durably applied) all changes up to the head of the stream at the time\n   * it was {@link aligned}. Never resolves if the subscriber is closed first.\n   */\n  get caughtUp(): Promise<void> {\n    return this.#caughtUp.promise;\n  }\n\n  #trackBackfills(change: WatermarkedChange) {\n    if (this.#backfills) {\n      const ignored = this.#backfills.applySerialized(change);\n      if (ignored.length && this.#aligned) {\n        this.#onBackfillIgnored?.(ignored);\n      }\n    }\n  }\n\n  get watermark() {\n    return this.#watermark;\n  }\n\n  get acked() {\n    return this.#acked;\n  }\n\n  /**\n   * Whether the backlog of live changes buffered during catchup has reached the\n   * point at which {@link send()} stops resolving. Past it the subscriber is no\n   * longer free: it holds up every subsequent flush, and with no other\n   * subscriber to form a majority it stalls replication outright.\n   */\n  get backlogFull() {\n    return (\n      this.#bufferedBacklogBytes >= this.#backlogBackpressure.highWaterBytes\n    );\n  }\n\n  /**\n   * @returns whether the subscriber is currently sending backlogged messages,\n   *          vs caught up and sending the \"head\" of the replication stream.\n   */\n  isBacklogged() {\n    return this.#backlog !== null || this.#bufferedBacklogBytes > 0;\n  }\n\n  /**\n   * Resolves the first time {@link backlogFull} becomes true, so that a caller\n   * holding the subscriber in catchup can give up at the moment the subscriber\n   * starts costing replication rather than on a timer. Call cancel() when the\n   * wait is no longer needed so the subscriber does not retain it.\n   */\n  whenBacklogFull(): BacklogFullWait {\n    if (this.backlogFull) {\n      return {promise: promiseVoid, cancel() {}};\n    }\n    const r = resolver<void>();\n    this.#backlogFullWaiters.add(r);\n    return {\n      promise: r.promise,\n      cancel: () => {\n        this.#backlogFullWaiters.delete(r);\n      },\n    };\n  }\n\n  send(change: WatermarkedChange): Promise<void> {\n    const [watermark] = change;\n    if (watermark > this.#watermark) {\n      if (this.#backlog) {\n        // During catchup, buffer live changes behind the durable catchup stream.\n        // The returned promise applies backpressure if the buffered bytes cross\n        // the high water mark.\n        this.#pushBacklog(change);\n        return this.#maybeWaitForBacklogSpace();\n      }\n      return this.#sendChange(change);\n    }\n    return promiseVoid;\n  }\n\n  sendBatch(\n    changes: readonly WatermarkedChange[],\n    preSerialized?: PreSerializedBatch | undefined,\n  ): Promise<void> {\n    if (changes.length === 0) {\n      return promiseVoid;\n    }\n\n    if (\n      this.#wsBatched &&\n      preSerialized &&\n      !this.#backlog &&\n      this.#initialized &&\n      changes[0][0] > this.#watermark &&\n      (this.#protocolVersion >= 5 ||\n        changes.every(c => this.supportsMessage(c[1])))\n    ) {\n      let commitWatermark: string | undefined;\n      for (let i = changes.length - 1; i >= 0; i--) {\n        if (changes[i][1] === 'commit') {\n          commitWatermark = changes[i][0];\n          break;\n        }\n      }\n      if (commitWatermark) {\n        this.#watermark = commitWatermark;\n      }\n      for (const change of changes) {\n        this.#trackBackfills(change);\n      }\n      return this.#sendPreSerializedDownstream(preSerialized, commitWatermark);\n    }\n\n    const promises: Promise<void>[] = [];\n    for (const change of changes) {\n      const p = this.send(change);\n      if (p !== promiseVoid) {\n        promises.push(p);\n      }\n    }\n    if (promises.length === 0) {\n      return promiseVoid;\n    }\n    if (promises.length === 1) {\n      return promises[0];\n    }\n    return Promise.all(promises).then(() => {});\n  }\n\n  #initialized = false;\n\n  /**\n   * Called once the subscriber's watermark has been validated in the initial\n   * catchup process.\n   */\n  #initialize() {\n    if (!this.#initialized) {\n      this.#initialized = true;\n      // The initial status precedes the catchup, so it bypasses the backlog.\n      void this.#sendDownstream(['status', {tag: 'status'}]);\n    }\n  }\n\n  /**\n   * Sends a status message (e.g. a lag report) downstream. While the\n   * subscriber is catching up, the status is queued in the backlog behind\n   * the changes that preceded it in the replication stream, so that the\n   * subscriber processes it in stream order. This is necessary for lag\n   * reports to measure the lag of the subscriber, including its catchup.\n   *\n   * Status messages received before the subscriber is initialized are\n   * dropped.\n   */\n  sendStatus(status: Status) {\n    if (!this.#initialized) {\n      return;\n    }\n    const downstream: Downstream = ['status', status];\n    if (this.#backlog) {\n      this.#pushBacklog({status: BigIntJSON.stringify(downstream)});\n      return;\n    }\n    void this.#sendDownstream(downstream);\n  }\n\n  /**\n   * Sends a control status (e.g. `draining` or `restore`) downstream\n   * immediately, regardless of whether the subscriber is initialized or\n   * catching up. Unlike {@link sendStatus}, such statuses are not tied to a\n   * position in the replication stream.\n   */\n  sendControlStatus(status: Status) {\n    void this.#sendDownstream(['status', status]);\n  }\n\n  /** catchup() is called on ChangeEntries loaded from the store. */\n  async catchup(change: WatermarkedChange) {\n    this.#initialize();\n    await this.#sendChange(change);\n  }\n\n  /**\n   * Marks the Subscribe as \"caught up\" and flushes any backlog of\n   * entries that were received during the catchup.\n   */\n  setCaughtUp(): Promise<void> {\n    this.#initialize();\n    if (!this.#backlog) {\n      return this.#backlogDrain ?? promiseVoid;\n    }\n    if (!this.#backlogDrain) {\n      // Keep #backlog non-null while queued entries are being handed to\n      // downstream. That preserves ordering for sends that race with\n      // setCaughtUp(): they append to the same backlog instead of bypassing\n      // older buffered changes.\n      this.#backlogDrain = this.#drainBacklog();\n      void this.#backlogDrain.catch(e => this.fail(e));\n    }\n    return this.#backlogDrain;\n  }\n\n  async #sendChange(change: WatermarkedChange) {\n    const [watermark, tag, json] = change;\n    if (watermark <= this.watermark) {\n      return;\n    }\n    if (!this.supportsMessage(tag)) {\n      return;\n    }\n    if (tag === 'commit') {\n      this.#watermark = watermark;\n    }\n    this.#trackBackfills(change);\n    const result = await this.#sendStringifiedDownstream(json);\n    if (tag === 'commit' && result === 'consumed') {\n      this.#advanceAck(watermark);\n    }\n  }\n\n  #advanceAck(watermark: string) {\n    // Sends can complete out of order (e.g. the bounded window in\n    // #drainBacklog), so the ack only advances monotonically, and listeners\n    // are only notified when it does.\n    const acked = max(this.#acked, watermark);\n    if (acked !== this.#acked) {\n      this.#acked = acked;\n      this.#onAck?.(acked);\n      this.#checkCaughtUp();\n    }\n  }\n\n  #checkCaughtUp() {\n    if (\n      this.#caughtUpWatermark !== undefined &&\n      this.#acked >= this.#caughtUpWatermark\n    ) {\n      this.#caughtUpWatermark = undefined;\n      this.#caughtUp.resolve();\n    }\n  }\n\n  #sendDownstream(downstream: Downstream) {\n    return this.#sendStringifiedDownstream(BigIntJSON.stringify(downstream));\n  }\n\n  async #sendStringifiedDownstream(json: string) {\n    const size = json.length;\n    this.#pending++;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(json);\n    try {\n      return await result;\n    } finally {\n      this.#pending--;\n      this.#pendingBytes -= size;\n      this.#processed++;\n    }\n  }\n\n  async #sendPreSerializedDownstream(\n    batch: PreSerializedBatch,\n    commitWatermark?: string | undefined,\n  ): Promise<void> {\n    const size = batch.byteLength;\n    this.#pending += batch.changes.length;\n    this.#pendingBytes += size;\n    const {result} = this.#downstream.push(batch);\n    try {\n      const outcome = await result;\n      if (commitWatermark && outcome === 'consumed') {\n        this.#advanceAck(commitWatermark);\n      }\n    } finally {\n      this.#pending -= batch.changes.length;\n      this.#pendingBytes -= size;\n      this.#processed += batch.changes.length;\n    }\n  }\n\n  // `pending` and `processed` stats are tracked by periodically sampling\n  // the running totals (by the progress tracker in the Forwarder).\n  // This information was originally collected for use in flow control\n  // decisions. The final flow control algorithm ended up being simpler\n  // than expected and does not actually use this information. However, the\n  // stats are still tracked and logged during flow control decisions for\n  // debugging, forensics, and potential improvements to the algorithm.\n\n  #pending = 0;\n  #pendingBytes = 0;\n  #processed = 0;\n  #samples: {processed: number; timestamp: number}[] = [\n    {processed: 0, timestamp: performance.now()},\n  ];\n\n  /**\n   * The number of downstream messages that have yet to be acked.\n   */\n  get numPending() {\n    return this.#pending + this.#backlogCount;\n  }\n\n  /**\n   * The total number of downstream messages that the subscriber has\n   * processed (i.e. acked).\n   */\n  get numProcessed() {\n    return this.#processed;\n  }\n\n  /**\n   * Records a new history entry for the number of messages processed,\n   * keeping the number of samples bounded to `maxSamples`.\n   */\n  sampleProcessRate(now: number, maxSamples = 10): this {\n    while (this.#samples.length >= maxSamples) {\n      this.#samples.shift();\n    }\n    this.#samples.push({processed: this.#processed, timestamp: now});\n    return this;\n  }\n\n  getStats(): SubscriberStats {\n    const pending = this.numPending;\n    if (this.#samples.length < 2) {\n      return {\n        processRate: 0,\n        pending,\n        backlog: this.#backlogCount,\n        backlogBytes: this.#bufferedBacklogBytes,\n        totalBufferedBytes: this.#totalBufferedBytes,\n        missedLastTimeout: this.#missedLastTimeout,\n      };\n    }\n    const from = this.#samples[0];\n    const to = must(this.#samples.at(-1));\n    const processed = to.processed - from.processed;\n    const seconds = (to.timestamp - from.timestamp) / 1000;\n    const processRate = seconds === 0 ? 0 : processed / seconds;\n    return {\n      processRate,\n      pending,\n      backlog: this.#backlogCount,\n      backlogBytes: this.#bufferedBacklogBytes,\n      totalBufferedBytes: this.#totalBufferedBytes,\n      missedLastTimeout: this.#missedLastTimeout,\n    };\n  }\n\n  #missedLastTimeout = false;\n  #laggingSinceMs: number | undefined;\n\n  trackResponseResult(result: 'on-time' | 'timed-out') {\n    this.#missedLastTimeout = result === 'timed-out';\n    if (result === 'on-time') {\n      this.#laggingSinceMs = undefined;\n    }\n  }\n\n  /**\n   * Reports the change rate of a slow subscriber (i.e. that missed the last\n   * timeout) compared to the change rate of the (slowest) subscriber that\n   * responded on time.\n   *\n   * * `lagging` indicates that this subscriber is slower\n   * * `catching-up` indicates that it is faster\n   *\n   * Returns the total duration in which subscriber has been continuously\n   * reported as `lagging`.\n   */\n  reportChangeRate(now: number, status: 'lagging' | 'catching-up') {\n    assert(\n      this.#missedLastTimeout,\n      `reportChangeRate should only be called for slow subscribers`,\n    );\n    if (status === 'catching-up') {\n      this.#laggingSinceMs = undefined;\n      return 0;\n    }\n    this.#laggingSinceMs ??= now;\n    return now - this.#laggingSinceMs;\n  }\n\n  supportsMessage(tag: ChangeTag) {\n    switch (tag) {\n      case 'update-table-metadata':\n        // update-table-row-key is only understood by subscribers >= protocol v5\n        return this.#protocolVersion >= 5;\n    }\n    return true;\n  }\n\n  /**\n   * Ends the subscription without sending a downstream `['error', ...]`.\n   *\n   * This is deliberate, and not the same as reporting the failure to the\n   * subscriber: `IncrementalSyncer` treats any `['error', ...]` as terminal and\n   * shuts down to restore a fresh replica from litestream, whereas a clean end\n   * backs off and re-subscribes. The failures routed here -- storer catchup\n   * errors, backlog drain errors, backup-monitor errors -- are transient, so a\n   * reconnect is the proportionate response and a fleet-wide restore is not.\n   *\n   * Callers are responsible for logging `err`; it is not carried downstream.\n   */\n  fail(_err?: unknown) {\n    this.close();\n  }\n\n  close(\n    error?: ErrorType,\n    message?: string,\n    details?: {olderReplica?: boolean | undefined},\n  ) {\n    // Closing the subscriber must also release producers that are blocked on\n    // backlog capacity; there is no future drain that could wake them.\n    this.#backlog = null;\n    this.#backlogBytes = 0;\n    this.#backlogBackpressure.releaseAll();\n    // The backlog can no longer grow, so nothing would ever resolve these.\n    // Waiters re-check backlogFull, which is now false, and see the close.\n    this.#resolveBacklogFullWaiters();\n\n    if (error !== undefined) {\n      // Wait for the ACK of the error message before closing the connection.\n      void this.#sendDownstream([\n        'error',\n        {type: error, message, ...details},\n      ]).finally(() => this.#downstream.cancel());\n    } else {\n      this.#downstream.cancel();\n    }\n  }\n\n  get #backlogCount() {\n    return this.#backlog?.size ?? 0;\n  }\n\n  get #bufferedBacklogBytes() {\n    // Include entries already handed to downstream but not yet consumed. Without\n    // this, setCaughtUp() could move bytes out of #backlog faster than the\n    // downstream Subscription can process them and release producers too early.\n    return this.#backlogBytes + this.#backlogInFlightBytes;\n  }\n\n  get #totalBufferedBytes() {\n    return this.#bufferedBacklogBytes + this.#pendingBytes;\n  }\n\n  #pushBacklog(entry: BacklogEntry) {\n    assert(this.#backlog, 'cannot push to backlog after catchup completed');\n    this.#backlog.push(entry);\n    this.#backlogBytes += entryBytes(entry);\n    if (this.backlogFull) {\n      this.#resolveBacklogFullWaiters();\n    }\n  }\n\n  #resolveBacklogFullWaiters() {\n    const waiters = [...this.#backlogFullWaiters];\n    this.#backlogFullWaiters.clear();\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n\n  #maybeWaitForBacklogSpace(): Promise<void> {\n    return this.#backlogBackpressure.waitForSpace(this.#bufferedBacklogBytes);\n  }\n\n  async #drainBacklog() {\n    const inFlight: {promise: Promise<void>; bytes: number}[] = [];\n    let inFlightBytes = 0;\n\n    try {\n      for (;;) {\n        const entry = this.#backlog?.shift();\n        if (!entry) {\n          const closed = this.#backlog === null;\n          this.#backlog = null;\n          this.#backlogBytes = 0;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n          if (!closed && !this.#aligned) {\n            // All changes up to the head have been handed to #sendChange()\n            // (and thus tracked), and subsequent changes are sent directly.\n            this.#aligned = true;\n            this.#onAligned?.(this);\n            this.#caughtUpWatermark = this.#watermark;\n            this.#checkCaughtUp();\n            void this.#lagReports?.then(lagReports => {\n              if (lagReports) {\n                lagReports.request();\n              } else {\n                this.sendStatus(LAG_REPORTING_DISABLED);\n              }\n            });\n          }\n          break;\n        }\n\n        const bytes = entryBytes(entry);\n        this.#backlogBytes -= bytes;\n        this.#backlogInFlightBytes += bytes;\n        this.#backlogBackpressure.releaseIfUnderLowWater(\n          this.#bufferedBacklogBytes,\n        );\n\n        // Send backlog entries in order, but keep only a bounded byte window in\n        // flight. This avoids replacing one unbounded buffer with another inside\n        // the downstream Subscription during catchup completion.\n        const send = isQueuedStatus(entry)\n          ? this.#sendStringifiedDownstream(entry.status).then(() => {})\n          : this.#sendChange(entry);\n        const promise = send.finally(() => {\n          this.#backlogInFlightBytes -= bytes;\n          this.#backlogBackpressure.releaseIfUnderLowWater(\n            this.#bufferedBacklogBytes,\n          );\n        });\n        inFlight.push({promise, bytes});\n        inFlightBytes += bytes;\n\n        while (inFlightBytes >= this.#backlogBackpressure.highWaterBytes) {\n          const next = must(inFlight.shift());\n          await next.promise;\n          inFlightBytes -= next.bytes;\n        }\n      }\n\n      for (const {promise} of inFlight) {\n        await promise;\n      }\n    } finally {\n      this.#backlogDrain = null;\n      this.#backlogBackpressure.releaseIfUnderLowWater(\n        this.#bufferedBacklogBytes,\n      );\n    }\n  }\n}\n\nclass ByteBackpressureGate {\n  readonly highWaterBytes: number;\n  readonly #lowWaterBytes: number;\n  readonly #waiters: Resolver<void>[] = [];\n\n  constructor(highWaterBytes: number, lowWaterRatio: number) {\n    this.highWaterBytes = Math.max(1, highWaterBytes);\n    this.#lowWaterBytes =\n      this.highWaterBytes * Math.min(1, Math.max(0, lowWaterRatio));\n  }\n\n  waitForSpace(bufferedBytes: number): Promise<void> {\n    if (bufferedBytes < this.highWaterBytes) {\n      return promiseVoid;\n    }\n\n    // One waiter represents one send() call that has already appended its\n    // change. The producer is released when the backlog falls back below the low\n    // water mark or the subscriber closes.\n    const r = resolver<void>();\n    this.#waiters.push(r);\n    return r.promise;\n  }\n\n  releaseIfUnderLowWater(bufferedBytes: number) {\n    if (this.#waiters.length === 0 || bufferedBytes > this.#lowWaterBytes) {\n      return;\n    }\n\n    // Use a low water mark so waiting producers are released in batches instead\n    // of waking one at a time around the high water boundary.\n    this.releaseAll();\n  }\n\n  releaseAll() {\n    const waiters = this.#waiters.splice(0);\n    for (const waiter of waiters) {\n      waiter.resolve();\n    }\n  }\n}\n\n/** Requests lag reports from upstream (see {@link SubscriberOptions.lagReports}). */\nexport type LagReports = {\n  /**\n   * Requests that a lag report be sent now rather than at the next report\n   * interval. Requests are coalesced upstream, e.g. with a report in flight.\n   */\n  request(): void;\n};\n\n/**\n * Sent to an aligned subscriber when lag reporting is disabled upstream,\n * signaling that the subscriber is caught up. This is a lag report with a\n * `nextSendTimeMs` of 0 and no `lastTimings`, so that no lag is measured\n * (see `isLagReportingDisabledSignal()` in the replicator's recorder).\n */\nexport const LAG_REPORTING_DISABLED: Status = {\n  tag: 'status',\n  lagReport: {nextSendTimeMs: 0},\n};\n"],"mappings":";;;;;;;;AAsBA,IAAM,mCAAmC;AACzC,IAAM,kCAAkC;AA8DxC,SAAS,eAAe,OAA4C;CAClE,OAAO,CAAC,MAAM,QAAQ,KAAK;AAC7B;AAEA,SAAS,WAAW,OAA6B;CAC/C,OAAO,eAAe,KAAK,IAAI,MAAM,OAAO,SAAS,MAAM,EAAE,CAAC;AAChE;;;;;;;;AAuBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA,gBAAgB;CAChB,wBAAwB;CACxB,gBAAsC;CACtC;CACA,sCAA+B,IAAI,IAAoB;CACvD;CACA;CACA;CACA;CACA;CACA,WAAW;CAGX;CACA,YAAqB,SAAe;CAEpC,YACE,iBACA,IACA,MACA,WACA,YACA,UAA6B,CAAC,GAC9B;EACA,KAAK,mBAAmB;EACxB,KAAK,KAAK;EACV,KAAK,OAAO;EACZ,KAAK,YAAY,QAAQ,aAAa;EACtC,KAAK,cAAc;EACnB,KAAK,aAAa,QAAQ,aAAa;EACvC,KAAK,aAAa;EAClB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,WAAW;EAC/B,KAAK,uBAAuB,IAAI,qBAC9B,QAAQ,yBAAyB,kCACjC,QAAQ,wBAAwB,+BAClC;EACA,KAAK,SAAS,QAAQ;EACtB,KAAK,aAAa,QAAQ;EAC1B,KAAK,aAAa,QAAQ;EAC1B,KAAK,qBAAqB,QAAQ;EAClC,KAAK,cAAc,QAAQ;CAC7B;;;;;CAMA,IAAI,YAAuC;EACzC,OAAO,KAAK;CACd;;;;;CAMA,IAAI,UAAmB;EACrB,OAAO,KAAK;CACd;;;;;;CAOA,IAAI,WAA0B;EAC5B,OAAO,KAAK,UAAU;CACxB;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAK,YAAY;GACnB,MAAM,UAAU,KAAK,WAAW,gBAAgB,MAAM;GACtD,IAAI,QAAQ,UAAU,KAAK,UACzB,KAAK,qBAAqB,OAAO;EAErC;CACF;CAEA,IAAI,YAAY;EACd,OAAO,KAAK;CACd;CAEA,IAAI,QAAQ;EACV,OAAO,KAAK;CACd;;;;;;;CAQA,IAAI,cAAc;EAChB,OACE,KAAK,yBAAyB,KAAK,qBAAqB;CAE5D;;;;;CAMA,eAAe;EACb,OAAO,KAAK,aAAa,QAAQ,KAAK,wBAAwB;CAChE;;;;;;;CAQA,kBAAmC;EACjC,IAAI,KAAK,aACP,OAAO;GAAC,SAAS;GAAa,SAAS,CAAC;EAAC;EAE3C,MAAM,IAAI,SAAe;EACzB,KAAK,oBAAoB,IAAI,CAAC;EAC9B,OAAO;GACL,SAAS,EAAE;GACX,cAAc;IACZ,KAAK,oBAAoB,OAAO,CAAC;GACnC;EACF;CACF;CAEA,KAAK,QAA0C;EAC7C,MAAM,CAAC,aAAa;EACpB,IAAI,YAAY,KAAK,YAAY;GAC/B,IAAI,KAAK,UAAU;IAIjB,KAAK,aAAa,MAAM;IACxB,OAAO,KAAK,0BAA0B;GACxC;GACA,OAAO,KAAK,YAAY,MAAM;EAChC;EACA,OAAO;CACT;CAEA,UACE,SACA,eACe;EACf,IAAI,QAAQ,WAAW,GACrB,OAAO;EAGT,IACE,KAAK,cACL,iBACA,CAAC,KAAK,YACN,KAAK,gBACL,QAAQ,EAAE,CAAC,KAAK,KAAK,eACpB,KAAK,oBAAoB,KACxB,QAAQ,OAAM,MAAK,KAAK,gBAAgB,EAAE,EAAE,CAAC,IAC/C;GACA,IAAI;GACJ,KAAK,IAAI,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KACvC,IAAI,QAAQ,EAAE,CAAC,OAAO,UAAU;IAC9B,kBAAkB,QAAQ,EAAE,CAAC;IAC7B;GACF;GAEF,IAAI,iBACF,KAAK,aAAa;GAEpB,KAAK,MAAM,UAAU,SACnB,KAAK,gBAAgB,MAAM;GAE7B,OAAO,KAAK,6BAA6B,eAAe,eAAe;EACzE;EAEA,MAAM,WAA4B,CAAC;EACnC,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,IAAI,KAAK,KAAK,MAAM;GAC1B,IAAI,MAAM,aACR,SAAS,KAAK,CAAC;EAEnB;EACA,IAAI,SAAS,WAAW,GACtB,OAAO;EAET,IAAI,SAAS,WAAW,GACtB,OAAO,SAAS;EAElB,OAAO,QAAQ,IAAI,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC;CAC5C;CAEA,eAAe;;;;;CAMf,cAAc;EACZ,IAAI,CAAC,KAAK,cAAc;GACtB,KAAK,eAAe;GAEpB,KAAU,gBAAgB,CAAC,UAAU,EAAC,KAAK,SAAQ,CAAC,CAAC;EACvD;CACF;;;;;;;;;;;CAYA,WAAW,QAAgB;EACzB,IAAI,CAAC,KAAK,cACR;EAEF,MAAM,aAAyB,CAAC,UAAU,MAAM;EAChD,IAAI,KAAK,UAAU;GACjB,KAAK,aAAa,EAAC,QAAQ,WAAW,UAAU,UAAU,EAAC,CAAC;GAC5D;EACF;EACA,KAAU,gBAAgB,UAAU;CACtC;;;;;;;CAQA,kBAAkB,QAAgB;EAChC,KAAU,gBAAgB,CAAC,UAAU,MAAM,CAAC;CAC9C;;CAGA,MAAM,QAAQ,QAA2B;EACvC,KAAK,YAAY;EACjB,MAAM,KAAK,YAAY,MAAM;CAC/B;;;;;CAMA,cAA6B;EAC3B,KAAK,YAAY;EACjB,IAAI,CAAC,KAAK,UACR,OAAO,KAAK,iBAAiB;EAE/B,IAAI,CAAC,KAAK,eAAe;GAKvB,KAAK,gBAAgB,KAAK,cAAc;GACxC,KAAU,cAAc,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;EACjD;EACA,OAAO,KAAK;CACd;CAEA,MAAM,YAAY,QAA2B;EAC3C,MAAM,CAAC,WAAW,KAAK,QAAQ;EAC/B,IAAI,aAAa,KAAK,WACpB;EAEF,IAAI,CAAC,KAAK,gBAAgB,GAAG,GAC3B;EAEF,IAAI,QAAQ,UACV,KAAK,aAAa;EAEpB,KAAK,gBAAgB,MAAM;EAC3B,MAAM,SAAS,MAAM,KAAK,2BAA2B,IAAI;EACzD,IAAI,QAAQ,YAAY,WAAW,YACjC,KAAK,YAAY,SAAS;CAE9B;CAEA,YAAY,WAAmB;EAI7B,MAAM,QAAQ,IAAI,KAAK,QAAQ,SAAS;EACxC,IAAI,UAAU,KAAK,QAAQ;GACzB,KAAK,SAAS;GACd,KAAK,SAAS,KAAK;GACnB,KAAK,eAAe;EACtB;CACF;CAEA,iBAAiB;EACf,IACE,KAAK,uBAAuB,KAAA,KAC5B,KAAK,UAAU,KAAK,oBACpB;GACA,KAAK,qBAAqB,KAAA;GAC1B,KAAK,UAAU,QAAQ;EACzB;CACF;CAEA,gBAAgB,YAAwB;EACtC,OAAO,KAAK,2BAA2B,WAAW,UAAU,UAAU,CAAC;CACzE;CAEA,MAAM,2BAA2B,MAAc;EAC7C,MAAM,OAAO,KAAK;EAClB,KAAK;EACL,KAAK,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK,IAAI;EAC3C,IAAI;GACF,OAAO,MAAM;EACf,UAAU;GACR,KAAK;GACL,KAAK,iBAAiB;GACtB,KAAK;EACP;CACF;CAEA,MAAM,6BACJ,OACA,iBACe;EACf,MAAM,OAAO,MAAM;EACnB,KAAK,YAAY,MAAM,QAAQ;EAC/B,KAAK,iBAAiB;EACtB,MAAM,EAAC,WAAU,KAAK,YAAY,KAAK,KAAK;EAC5C,IAAI;GACF,MAAM,UAAU,MAAM;GACtB,IAAI,mBAAmB,YAAY,YACjC,KAAK,YAAY,eAAe;EAEpC,UAAU;GACR,KAAK,YAAY,MAAM,QAAQ;GAC/B,KAAK,iBAAiB;GACtB,KAAK,cAAc,MAAM,QAAQ;EACnC;CACF;CAUA,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,IAAI;CAAC,CAC7C;;;;CAKA,IAAI,aAAa;EACf,OAAO,KAAK,WAAW,KAAK;CAC9B;;;;;CAMA,IAAI,eAAe;EACjB,OAAO,KAAK;CACd;;;;;CAMA,kBAAkB,KAAa,aAAa,IAAU;EACpD,OAAO,KAAK,SAAS,UAAU,YAC7B,KAAK,SAAS,MAAM;EAEtB,KAAK,SAAS,KAAK;GAAC,WAAW,KAAK;GAAY,WAAW;EAAG,CAAC;EAC/D,OAAO;CACT;CAEA,WAA4B;EAC1B,MAAM,UAAU,KAAK;EACrB,IAAI,KAAK,SAAS,SAAS,GACzB,OAAO;GACL,aAAa;GACb;GACA,SAAS,KAAK;GACd,cAAc,KAAK;GACnB,oBAAoB,KAAK;GACzB,mBAAmB,KAAK;EAC1B;EAEF,MAAM,OAAO,KAAK,SAAS;EAC3B,MAAM,KAAK,KAAK,KAAK,SAAS,GAAG,EAAE,CAAC;EACpC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;EAElD,OAAO;GACL,aAFkB,YAAY,IAAI,IAAI,YAAY;GAGlD;GACA,SAAS,KAAK;GACd,cAAc,KAAK;GACnB,oBAAoB,KAAK;GACzB,mBAAmB,KAAK;EAC1B;CACF;CAEA,qBAAqB;CACrB;CAEA,oBAAoB,QAAiC;EACnD,KAAK,qBAAqB,WAAW;EACrC,IAAI,WAAW,WACb,KAAK,kBAAkB,KAAA;CAE3B;;;;;;;;;;;;CAaA,iBAAiB,KAAa,QAAmC;EAC/D,OACE,KAAK,oBACL,6DACF;EACA,IAAI,WAAW,eAAe;GAC5B,KAAK,kBAAkB,KAAA;GACvB,OAAO;EACT;EACA,KAAK,oBAAoB;EACzB,OAAO,MAAM,KAAK;CACpB;CAEA,gBAAgB,KAAgB;EAC9B,QAAQ,KAAR;GACE,KAAK,yBAEH,OAAO,KAAK,oBAAoB;EACpC;EACA,OAAO;CACT;;;;;;;;;;;;;CAcA,KAAK,MAAgB;EACnB,KAAK,MAAM;CACb;CAEA,MACE,OACA,SACA,SACA;EAGA,KAAK,WAAW;EAChB,KAAK,gBAAgB;EACrB,KAAK,qBAAqB,WAAW;EAGrC,KAAK,2BAA2B;EAEhC,IAAI,UAAU,KAAA,GAEZ,KAAU,gBAAgB,CACxB,SACA;GAAC,MAAM;GAAO;GAAS,GAAG;EAAO,CACnC,CAAC,CAAC,CAAC,cAAc,KAAK,YAAY,OAAO,CAAC;OAE1C,KAAK,YAAY,OAAO;CAE5B;CAEA,IAAI,gBAAgB;EAClB,OAAO,KAAK,UAAU,QAAQ;CAChC;CAEA,IAAI,wBAAwB;EAI1B,OAAO,KAAK,gBAAgB,KAAK;CACnC;CAEA,IAAI,sBAAsB;EACxB,OAAO,KAAK,wBAAwB,KAAK;CAC3C;CAEA,aAAa,OAAqB;EAChC,OAAO,KAAK,UAAU,gDAAgD;EACtE,KAAK,SAAS,KAAK,KAAK;EACxB,KAAK,iBAAiB,WAAW,KAAK;EACtC,IAAI,KAAK,aACP,KAAK,2BAA2B;CAEpC;CAEA,6BAA6B;EAC3B,MAAM,UAAU,CAAC,GAAG,KAAK,mBAAmB;EAC5C,KAAK,oBAAoB,MAAM;EAC/B,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;CAEA,4BAA2C;EACzC,OAAO,KAAK,qBAAqB,aAAa,KAAK,qBAAqB;CAC1E;CAEA,MAAM,gBAAgB;EACpB,MAAM,WAAsD,CAAC;EAC7D,IAAI,gBAAgB;EAEpB,IAAI;GACF,SAAS;IACP,MAAM,QAAQ,KAAK,UAAU,MAAM;IACnC,IAAI,CAAC,OAAO;KACV,MAAM,SAAS,KAAK,aAAa;KACjC,KAAK,WAAW;KAChB,KAAK,gBAAgB;KACrB,KAAK,qBAAqB,uBACxB,KAAK,qBACP;KACA,IAAI,CAAC,UAAU,CAAC,KAAK,UAAU;MAG7B,KAAK,WAAW;MAChB,KAAK,aAAa,IAAI;MACtB,KAAK,qBAAqB,KAAK;MAC/B,KAAK,eAAe;MACpB,KAAU,aAAa,MAAK,eAAc;OACxC,IAAI,YACF,WAAW,QAAQ;YAEnB,KAAK,WAAW,sBAAsB;MAE1C,CAAC;KACH;KACA;IACF;IAEA,MAAM,QAAQ,WAAW,KAAK;IAC9B,KAAK,iBAAiB;IACtB,KAAK,yBAAyB;IAC9B,KAAK,qBAAqB,uBACxB,KAAK,qBACP;IAQA,MAAM,WAHO,eAAe,KAAK,IAC7B,KAAK,2BAA2B,MAAM,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC,IAC3D,KAAK,YAAY,KAAK,EAAA,CACL,cAAc;KACjC,KAAK,yBAAyB;KAC9B,KAAK,qBAAqB,uBACxB,KAAK,qBACP;IACF,CAAC;IACD,SAAS,KAAK;KAAC;KAAS;IAAK,CAAC;IAC9B,iBAAiB;IAEjB,OAAO,iBAAiB,KAAK,qBAAqB,gBAAgB;KAChE,MAAM,OAAO,KAAK,SAAS,MAAM,CAAC;KAClC,MAAM,KAAK;KACX,iBAAiB,KAAK;IACxB;GACF;GAEA,KAAK,MAAM,EAAC,aAAY,UACtB,MAAM;EAEV,UAAU;GACR,KAAK,gBAAgB;GACrB,KAAK,qBAAqB,uBACxB,KAAK,qBACP;EACF;CACF;AACF;AAEA,IAAM,uBAAN,MAA2B;CACzB;CACA;CACA,WAAsC,CAAC;CAEvC,YAAY,gBAAwB,eAAuB;EACzD,KAAK,iBAAiB,KAAK,IAAI,GAAG,cAAc;EAChD,KAAK,iBACH,KAAK,iBAAiB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,aAAa,CAAC;CAChE;CAEA,aAAa,eAAsC;EACjD,IAAI,gBAAgB,KAAK,gBACvB,OAAO;EAMT,MAAM,IAAI,SAAe;EACzB,KAAK,SAAS,KAAK,CAAC;EACpB,OAAO,EAAE;CACX;CAEA,uBAAuB,eAAuB;EAC5C,IAAI,KAAK,SAAS,WAAW,KAAK,gBAAgB,KAAK,gBACrD;EAKF,KAAK,WAAW;CAClB;CAEA,aAAa;EACX,MAAM,UAAU,KAAK,SAAS,OAAO,CAAC;EACtC,KAAK,MAAM,UAAU,SACnB,OAAO,QAAQ;CAEnB;AACF;;;;;;;AAiBA,IAAa,yBAAiC;CAC5C,KAAK;CACL,WAAW,EAAC,gBAAgB,EAAC;AAC/B"}

package/out/zero-cache/src/services/heapz.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"heapz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/heapz.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAC1D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AAGjE,wBAAgB,kBAAkB,CAChC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,QA0BlB"}
+{"version":3,"file":"heapz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/heapz.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAC1D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AAGjE,wBAAgB,kBAAkB,CAChC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,QAkClB"}

package/out/zero-cache/src/services/heapz.js

@@ -1,14 +1,13 @@
-import { isAdminPasswordValid } from "../config/zero-config.js";
+import { authorizeAdminRequest } from "./admin-auth.js";
 import fs from "fs";
 import v8 from "v8";
-import auth from "basic-auth";
 //#region ../zero-cache/src/services/heapz.ts
 function handleHeapzRequest(lc, config, req, res) {
-	const credentials = auth(req);
-	if (!isAdminPasswordValid(lc, config, credentials?.pass)) {
-		res.code(401).header("WWW-Authenticate", "Basic realm=\"Heapz Protected Area\"").send("Unauthorized");
+	if (!config.enableHeapz) {
+		res.code(404).send("Not Found");
 		return;
 	}
+	if (authorizeAdminRequest(lc, config, req, res, "Heapz Protected Area", "admin") === void 0) return;
 	const filename = v8.writeHeapSnapshot();
 	const stream = fs.createReadStream(filename);
 	res.header("Content-Type", "application/octet-stream").header("Content-Disposition", `attachment; filename=${filename}`).send(stream);

package/out/zero-cache/src/services/heapz.js.map

@@ -1 +1 @@
-{"version":3,"file":"heapz.js","names":[],"sources":["../../../../../zero-cache/src/services/heapz.ts"],"sourcesContent":["import fs from 'fs';\nimport v8 from 'v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\n\nexport function handleHeapzRequest(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  const credentials = auth(req);\n  if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n    void res\n      .code(401)\n      .header('WWW-Authenticate', 'Basic realm=\"Heapz Protected Area\"')\n      .send('Unauthorized');\n    return;\n  }\n\n  const filename = v8.writeHeapSnapshot();\n  const stream = fs.createReadStream(filename);\n  void res\n    .header('Content-Type', 'application/octet-stream')\n    .header('Content-Disposition', `attachment; filename=${filename}`)\n    .send(stream);\n\n  // Clean up temp file after streaming\n  stream.on('end', () => {\n    fs.unlink(filename, err => {\n      if (err) {\n        lc.error?.('Error deleting heap snapshot:', err);\n      }\n    });\n  });\n}\n"],"mappings":";;;;;AAQA,SAAgB,mBACd,IACA,QACA,KACA,KACA;CACA,MAAM,cAAc,KAAK,GAAG;CAC5B,IAAI,CAAC,qBAAqB,IAAI,QAAQ,aAAa,IAAI,GAAG;EACxD,IACG,KAAK,GAAG,CAAC,CACT,OAAO,oBAAoB,sCAAoC,CAAC,CAChE,KAAK,cAAc;EACtB;CACF;CAEA,MAAM,WAAW,GAAG,kBAAkB;CACtC,MAAM,SAAS,GAAG,iBAAiB,QAAQ;CAC3C,IACG,OAAO,gBAAgB,0BAA0B,CAAC,CAClD,OAAO,uBAAuB,wBAAwB,UAAU,CAAC,CACjE,KAAK,MAAM;CAGd,OAAO,GAAG,aAAa;EACrB,GAAG,OAAO,WAAU,QAAO;GACzB,IAAI,KACF,GAAG,QAAQ,iCAAiC,GAAG;EAEnD,CAAC;CACH,CAAC;AACH"}
+{"version":3,"file":"heapz.js","names":[],"sources":["../../../../../zero-cache/src/services/heapz.ts"],"sourcesContent":["import fs from 'fs';\nimport v8 from 'v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {authorizeAdminRequest} from './admin-auth.ts';\n\nexport function handleHeapzRequest(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  req: FastifyRequest,\n  res: FastifyReply,\n) {\n  if (!config.enableHeapz) {\n    void res.code(404).send('Not Found');\n    return;\n  }\n  if (\n    authorizeAdminRequest(\n      lc,\n      config,\n      req,\n      res,\n      'Heapz Protected Area',\n      'admin',\n    ) === undefined\n  ) {\n    return;\n  }\n\n  const filename = v8.writeHeapSnapshot();\n  const stream = fs.createReadStream(filename);\n  void res\n    .header('Content-Type', 'application/octet-stream')\n    .header('Content-Disposition', `attachment; filename=${filename}`)\n    .send(stream);\n\n  // Clean up temp file after streaming\n  stream.on('end', () => {\n    fs.unlink(filename, err => {\n      if (err) {\n        lc.error?.('Error deleting heap snapshot:', err);\n      }\n    });\n  });\n}\n"],"mappings":";;;;AAOA,SAAgB,mBACd,IACA,QACA,KACA,KACA;CACA,IAAI,CAAC,OAAO,aAAa;EACvB,IAAS,KAAK,GAAG,CAAC,CAAC,KAAK,WAAW;EACnC;CACF;CACA,IACE,sBACE,IACA,QACA,KACA,KACA,wBACA,OACF,MAAM,KAAA,GAEN;CAGF,MAAM,WAAW,GAAG,kBAAkB;CACtC,MAAM,SAAS,GAAG,iBAAiB,QAAQ;CAC3C,IACG,OAAO,gBAAgB,0BAA0B,CAAC,CAClD,OAAO,uBAAuB,wBAAwB,UAAU,CAAC,CACjE,KAAK,MAAM;CAGd,OAAO,GAAG,aAAa;EACrB,GAAG,OAAO,WAAU,QAAO;GACzB,IAAI,KACF,GAAG,QAAQ,iCAAiC,GAAG;EAEnD,CAAC;CACH,CAAC;AACH"}

package/out/zero-cache/src/services/mutagen/error.d.ts

@@ -1,4 +1,9 @@
 export declare class MutationAlreadyProcessedError extends Error {
+    /**
+     * The client's current last mutation ID. Kept out of the message, which is
+     * returned to the client; the server logs it separately.
+     */
+    readonly lastMutationID: number | bigint;
     constructor(clientID: string, received: number, actual: number | bigint);
 }
 //# sourceMappingURL=error.d.ts.map

package/out/zero-cache/src/services/mutagen/error.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"error.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/error.ts"],"names":[],"mappings":"AAEA,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,YAAY,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAStE;CACF"}
+{"version":3,"file":"error.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/error.ts"],"names":[],"mappings":"AAEA,qBAAa,6BAA8B,SAAQ,KAAK;IACtD;;;OAGG;IACH,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,MAAM,CAAC;IAEzC,YAAY,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAUtE;CACF"}

package/out/zero-cache/src/services/mutagen/error.js

@@ -1,8 +1,14 @@
 import { assert } from "../../../../shared/src/asserts.js";
 //#region ../zero-cache/src/services/mutagen/error.ts
 var MutationAlreadyProcessedError = class extends Error {
+	/**
+	* The client's current last mutation ID. Kept out of the message, which is
+	* returned to the client; the server logs it separately.
+	*/
+	lastMutationID;
 	constructor(clientID, received, actual) {
-		super(`Ignoring mutation from ${clientID} with ID ${received} as it was already processed. Expected: ${actual}`);
+		super(`Ignoring mutation from ${clientID} with ID ${received} as it was already processed`);
+		this.lastMutationID = actual;
 		assert(received < actual, () => `MutationAlreadyProcessedError: received (${received}) must be < actual (${actual})`);
 	}
 };

package/out/zero-cache/src/services/mutagen/error.js.map

@@ -1 +1 @@
-{"version":3,"file":"error.js","names":[],"sources":["../../../../../../zero-cache/src/services/mutagen/error.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\n\nexport class MutationAlreadyProcessedError extends Error {\n  constructor(clientID: string, received: number, actual: number | bigint) {\n    super(\n      `Ignoring mutation from ${clientID} with ID ${received} as it was already processed. Expected: ${actual}`,\n    );\n    assert(\n      received < actual,\n      () =>\n        `MutationAlreadyProcessedError: received (${received}) must be < actual (${actual})`,\n    );\n  }\n}\n"],"mappings":";;AAEA,IAAa,gCAAb,cAAmD,MAAM;CACvD,YAAY,UAAkB,UAAkB,QAAyB;EACvE,MACE,0BAA0B,SAAS,WAAW,SAAS,0CAA0C,QACnG;EACA,OACE,WAAW,cAET,4CAA4C,SAAS,sBAAsB,OAAO,EACtF;CACF;AACF"}
+{"version":3,"file":"error.js","names":[],"sources":["../../../../../../zero-cache/src/services/mutagen/error.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\n\nexport class MutationAlreadyProcessedError extends Error {\n  /**\n   * The client's current last mutation ID. Kept out of the message, which is\n   * returned to the client; the server logs it separately.\n   */\n  readonly lastMutationID: number | bigint;\n\n  constructor(clientID: string, received: number, actual: number | bigint) {\n    super(\n      `Ignoring mutation from ${clientID} with ID ${received} as it was already processed`,\n    );\n    this.lastMutationID = actual;\n    assert(\n      received < actual,\n      () =>\n        `MutationAlreadyProcessedError: received (${received}) must be < actual (${actual})`,\n    );\n  }\n}\n"],"mappings":";;AAEA,IAAa,gCAAb,cAAmD,MAAM;;;;;CAKvD;CAEA,YAAY,UAAkB,UAAkB,QAAyB;EACvE,MACE,0BAA0B,SAAS,WAAW,SAAS,6BACzD;EACA,KAAK,iBAAiB;EACtB,OACE,WAAW,cAET,4CAA4C,SAAS,sBAAsB,OAAO,EACtF;CACF;AACF"}

package/out/zero-cache/src/services/mutagen/mutagen.js

@@ -99,7 +99,7 @@ async function processMutation(lc, authData, db, shard, clientGroupID, mutation,
 			break;
 		} catch (e) {
 			if (e instanceof MutationAlreadyProcessedError) {
-				lc.debug?.(e.message);
+				lc.debug?.(`${e.message}. Expected mutation ID ${e.lastMutationID}`);
 				return;
 			}
 			if (isProtocolError(e) && !errorMode && e.kind === "InvalidPush" && customMutatorsEnabled && i < 2) {

package/out/zero-cache/src/services/mutagen/mutagen.js.map

@@ -1 +1 @@
-{"version":3,"file":"mutagen.js","names":["ErrorKind.MutationRateLimited","MutationType.CRUD","Mode.SERIALIZABLE","ErrorKind.MutationFailed","v.parse","ErrorKind.InvalidPush","ErrorOrigin.ZeroCache"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {resolver} from '../../../../shared/src/resolver.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  type CRUDMutation,\n  type DeleteOp,\n  type InsertOp,\n  type Mutation,\n  type UpdateOp,\n  type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n  primaryKeyValueSchema,\n  type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  WriteAuthorizerImpl,\n  type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n  kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n  desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n  /**\n   * Callers must hold a {@link ref} for the duration of a push (i.e. across\n   * all of its mutations). The service stops, closing its replica handle,\n   * when its ref count drops to zero.\n   */\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled: boolean,\n  ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #upstream: PostgresDB;\n  readonly #shard: ShardID;\n  readonly #stopped = resolver();\n  readonly #replica: Database;\n  readonly #writeAuthorizer: WriteAuthorizerImpl;\n  readonly #limiter: SlidingWindowLimiter | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  readonly #crudMutations = getOrCreateCounter(\n    'mutation',\n    'crud',\n    'Number of CRUD mutations processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    clientGroupID: string,\n    upstream: PostgresDB,\n    config: ZeroConfig,\n    writeAuthzStorage: DatabaseStorage,\n  ) {\n    this.id = clientGroupID;\n    this.#lc = lc;\n    this.#upstream = upstream;\n    this.#shard = shard;\n    this.#replica = new Database(this.#lc, config.replica.file, {\n      fileMustExist: true,\n    });\n    this.#writeAuthorizer = new WriteAuthorizerImpl(\n      this.#lc,\n      config,\n      this.#replica,\n      shard.appID,\n      clientGroupID,\n      writeAuthzStorage,\n    );\n\n    if (config.perUserMutationLimit.max !== undefined) {\n      this.#limiter = new SlidingWindowLimiter(\n        config.perUserMutationLimit.windowMs,\n        config.perUserMutationLimit.max,\n      );\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled = false,\n  ): Promise<MutationError | undefined> {\n    if (this.#limiter?.canDo() === false) {\n      return Promise.resolve([\n        ErrorKind.MutationRateLimited,\n        'Rate limit exceeded',\n      ]);\n    }\n    this.#crudMutations.add(1, {\n      clientGroupID: this.id,\n    });\n    return processMutation(\n      this.#lc,\n      authData,\n      this.#upstream,\n      this.#shard,\n      this.id,\n      mutation,\n      this.#writeAuthorizer,\n      undefined,\n      customMutatorsEnabled,\n    );\n  }\n\n  run(): Promise<void> {\n    return this.#stopped.promise;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return this.#stopped.promise;\n    }\n    this.#isStopped = true;\n    try {\n      this.#writeAuthorizer.destroy();\n    } catch (e) {\n      this.#lc.error?.('error destroying write authorizer storage', e);\n    } finally {\n      // The replica's statistics are maintained by the replicator, so skip\n      // the `PRAGMA optimize` that closing a writable handle would otherwise\n      // run on every client group churn.\n      this.#replica.close({optimize: false});\n    }\n    this.#stopped.resolve();\n    return this.#stopped.promise;\n  }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n  lc: LogContext,\n  authData: JWTPayload | undefined,\n  db: PostgresDB,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: Mutation,\n  writeAuthorizer: WriteAuthorizer,\n  onTxStart?: () => void | Promise<void>, // for testing\n  customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n  assert(\n    mutation.type === MutationType.CRUD,\n    'Only CRUD mutations are supported',\n  );\n  // log-leak-ignore -- the mutation id, not row data\n  lc = lc.withContext('mutationID', mutation.id);\n  lc = lc.withContext('processMutation');\n  // The mutationID context set above identifies it; the mutation itself\n  // carries the row values being written.\n  lc.debug?.('Process mutation start');\n\n  // Record mutation processing attempt for telemetry (regardless of success/failure)\n  recordMutation('crud');\n\n  let result: MutationError | undefined;\n\n  const start = Date.now();\n  try {\n    // Mutations can fail for a variety of reasons:\n    //\n    // - application error\n    // - network/db error\n    // - zero bug\n    //\n    // For application errors what we want is to re-run the mutation in\n    // \"error mode\", which skips the actual mutation and just updates the\n    // lastMutationID. Then return the error to the app.\n    //\n    // However, it's hard to tell the difference between application errors\n    // and the other types.\n    //\n    // A reasonable policy ends up being to just retry every mutation once\n    // in error mode. If the error mode mutation succeeds then we assume it\n    // was an application error and return the error to the app. Otherwise,\n    // we know it was something internal and we log it.\n    //\n    // This is not 100% correct - there are theoretical cases where we\n    // return an internal error to the app that shouldn't have been. But it\n    // would have to be a crazy coincidence: we'd have to have a network\n    // error on the first attempt that resolves by the second attempt.\n    //\n    // One might ask why not try/catch just the calls to the mutators and\n    // consider those application errors. That is actually what we do in\n    // Replicache:\n    //\n    // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n    //\n    // We don't do it here because:\n    //\n    // 1. It's still not perfect. It's hard to isolate SQL errors in\n    //    mutators due to app developer mistakes from SQL errors due to\n    //    Zero mistakes.\n    // 2. It's not possible to do this with the pg library we're using in\n    //    Zero anyway: https://github.com/porsager/postgres/issues/455.\n    //\n    // Personally I think this simple retry policy is nice.\n    let errorMode = false;\n    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n      try {\n        await runTx(\n          db,\n          async tx => {\n            // Simulates a concurrent request for testing. In production this is a noop.\n            const done = onTxStart?.();\n            try {\n              return await processMutationWithTx(\n                lc,\n                tx,\n                authData,\n                shard,\n                clientGroupID,\n                mutation,\n                errorMode,\n                writeAuthorizer,\n              );\n            } finally {\n              await done;\n            }\n          },\n          {mode: Mode.SERIALIZABLE},\n        );\n        if (errorMode) {\n          lc.debug?.('Ran mutation successfully in error mode');\n        }\n        break;\n      } catch (e) {\n        if (e instanceof MutationAlreadyProcessedError) {\n          lc.debug?.(e.message);\n          // Don't double-count already processed mutations, but they were counted above\n          return undefined;\n        }\n        if (\n          isProtocolError(e) &&\n          !errorMode &&\n          e.kind === ErrorKind.InvalidPush &&\n          customMutatorsEnabled &&\n          i < 2\n        ) {\n          // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n          // This can create a lot of OOO mutation errors since we do not know when the API server\n          // has applied a custom mutation before moving on to process CRUD mutations.\n          // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n          // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n          // They should migrate completely to custom mutators.\n          lc.info?.(\n            'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n            e,\n          );\n          await new Promise(resolve => setTimeout(resolve, 100));\n          continue;\n        }\n        if (isProtocolError(e) || errorMode) {\n          lc.error?.('Process mutation error', e);\n          throw e;\n        }\n        if (\n          e instanceof postgres.PostgresError &&\n          e.code === PG_SERIALIZATION_FAILURE\n        ) {\n          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n          continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n        }\n        result = [ErrorKind.MutationFailed, String(e)];\n        if (errorMode) {\n          break;\n        }\n        lc.error?.('Got error running mutation, re-running in error mode', e);\n        errorMode = true;\n        i--;\n      }\n    }\n  } finally {\n    lc.debug?.('Process mutation complete in', Date.now() - start);\n  }\n  return result;\n}\n\nasync function processMutationWithTx(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  authData: JWTPayload | undefined,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: CRUDMutation,\n  errorMode: boolean,\n  authorizer: WriteAuthorizer,\n) {\n  const tasks: (() => Promise<unknown>)[] = [];\n\n  async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n    try {\n      return await stmt.execute();\n    } finally {\n      const q = stmt as unknown as Query;\n      // `q.parameters` are the bound row values -- log the parameterized\n      // statement and the parameter count, never the values themselves.\n      lc.debug?.(`${q.string} (${q.parameters.length} params)`);\n    }\n  }\n\n  authorizer.reloadPermissions();\n\n  if (!errorMode) {\n    // Validates table names and keys every op by the replica's primary key,\n    // which is what the authorizer checks.\n    const ops = authorizer.validateOps(mutation.args[0].ops);\n\n    const normalizedOps = authorizer.normalizeOps(ops);\n    const [canPre, canPost] = await Promise.all([\n      authorizer.canPreMutation(authData, normalizedOps),\n      authorizer.canPostMutation(authData, normalizedOps),\n    ]);\n    if (canPre && canPost) {\n      for (const op of ops) {\n        switch (op.op) {\n          case 'insert':\n            tasks.push(() => execute(getInsertSQL(tx, op)));\n            break;\n          case 'upsert':\n            tasks.push(() => execute(getUpsertSQL(tx, op)));\n            break;\n          case 'update':\n            tasks.push(() => execute(getUpdateSQL(tx, op)));\n            break;\n          case 'delete':\n            tasks.push(() => execute(getDeleteSQL(tx, op)));\n            break;\n          default:\n            unreachable(op);\n        }\n      }\n    }\n  }\n\n  // Confirm the mutation even though it may have been blocked by the authorizer.\n  // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n  tasks.unshift(() =>\n    checkSchemaVersionAndIncrementLastMutationID(\n      tx,\n      shard,\n      clientGroupID,\n      mutation.clientID,\n      mutation.id,\n    ),\n  );\n\n  // Note: An error thrown from any Promise aborts the entire transaction.\n  await Promise.all(tasks.map(task => task()));\n}\n\nfunction getInsertSQL(\n  tx: postgres.TransactionSql,\n  create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  return tx`\n    INSERT INTO ${tx(create.tableName)} ${tx(create.value)}\n    ON CONFLICT (${tx(create.primaryKey)}) DO NOTHING\n  `;\n}\n\nfunction getUpsertSQL(\n  tx: postgres.TransactionSql,\n  set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = set;\n  return tx`\n    INSERT INTO ${tx(tableName)} ${tx(value)}\n    ON CONFLICT (${tx(primaryKey)})\n    DO UPDATE SET ${tx(value)}\n  `;\n}\n\nfunction getUpdateSQL(\n  tx: postgres.TransactionSql,\n  update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const table = update.tableName;\n  const {primaryKey, value} = update;\n  const id: Record<string, PrimaryKeyValue> = {};\n  for (const key of primaryKey) {\n    id[key] = v.parse(value[key], primaryKeyValueSchema);\n  }\n  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n    id,\n  ).flatMap(([key, value], i) =>\n    i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n  )}`;\n}\n\nfunction getDeleteSQL(\n  tx: postgres.TransactionSql,\n  deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = deleteOp;\n\n  const conditions = [];\n  for (const key of primaryKey) {\n    if (conditions.length > 0) {\n      conditions.push(tx`AND`);\n    }\n    conditions.push(tx`${tx(key)} = ${value[key]}`);\n  }\n\n  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n  tx: PostgresTransaction,\n  shard: ShardID,\n  clientGroupID: string,\n  clientID: string,\n  receivedMutationID: number,\n) {\n  const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n    INSERT INTO ${tx(upstreamSchema(shard))}.clients \n      as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n          VALUES (${clientGroupID}, ${clientID}, ${1})\n      ON CONFLICT (\"clientGroupID\", \"clientID\")\n      DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n      RETURNING \"lastMutationID\"\n  `;\n\n  // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n  if (receivedMutationID < lastMutationID) {\n    throw new MutationAlreadyProcessedError(\n      clientID,\n      receivedMutationID,\n      lastMutationID,\n    );\n  } else if (receivedMutationID > lastMutationID) {\n    throw new ProtocolError({\n      kind: ErrorKind.InvalidPush,\n      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n      origin: ErrorOrigin.ZeroCache,\n    });\n  }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,oCACF;CAEA,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;EACA,KAAK,KAAK;EACV,KAAK,MAAM;EACX,KAAK,YAAY;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,SAAS,KAAK,KAAK,OAAO,QAAQ,MAAM,EAC1D,eAAe,KACjB,CAAC;EACD,KAAK,mBAAmB,IAAI,oBAC1B,KAAK,KACL,QACA,KAAK,UACL,MAAM,OACN,eACA,iBACF;EAEA,IAAI,OAAO,qBAAqB,QAAQ,KAAA,GACtC,KAAK,WAAW,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,GAC9B;CAEJ;CAEA,MAAM;EACJ,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;EACP,IAAI,KAAK,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAK,YAAY;CAC1B;CAEA,gBACE,UACA,UACA,wBAAwB,OACY;EACpC,IAAI,KAAK,UAAU,MAAM,MAAM,OAC7B,OAAO,QAAQ,QAAQ,CACrBA,qBACA,qBACF,CAAC;EAEH,KAAK,eAAe,IAAI,GAAG,EACzB,eAAe,KAAK,GACtB,CAAC;EACD,OAAO,gBACL,KAAK,KACL,UACA,KAAK,WACL,KAAK,QACL,KAAK,IACL,UACA,KAAK,kBACL,KAAA,GACA,qBACF;CACF;CAEA,MAAqB;EACnB,OAAO,KAAK,SAAS;CACvB;CAEA,OAAsB;EACpB,IAAI,KAAK,YACP,OAAO,KAAK,SAAS;EAEvB,KAAK,aAAa;EAClB,IAAI;GACF,KAAK,iBAAiB,QAAQ;EAChC,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,6CAA6C,CAAC;EACjE,UAAU;GAIR,KAAK,SAAS,MAAM,EAAC,UAAU,MAAK,CAAC;EACvC;EACA,KAAK,SAAS,QAAQ;EACtB,OAAO,KAAK,SAAS;CACvB;AACF;AAEA,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;CACpC,OACE,SAAS,SAASC,MAClB,mCACF;CAEA,KAAK,GAAG,YAAY,cAAc,SAAS,EAAE;CAC7C,KAAK,GAAG,YAAY,iBAAiB;CAGrC,GAAG,QAAQ,wBAAwB;CAGnC,eAAe,MAAM;CAErB,IAAI;CAEJ,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EAuCF,IAAI,YAAY;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,KAC9C,IAAI;GACF,MAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,YAAY;IACzB,IAAI;KACF,OAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,eACF;IACF,UAAU;KACR,MAAM;IACR;GACF,GACA,EAAC,MAAMC,aAAiB,CAC1B;GACA,IAAI,WACF,GAAG,QAAQ,yCAAyC;GAEtD;EACF,SAAS,GAAG;GACV,IAAI,aAAa,+BAA+B;IAC9C,GAAG,QAAQ,EAAE,OAAO;IAEpB;GACF;GACA,IACE,gBAAgB,CAAC,KACjB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;IAOA,GAAG,OACD,kJACA,CACF;IACA,MAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,GAAG,CAAC;IACrD;GACF;GACA,IAAI,gBAAgB,CAAC,KAAK,WAAW;IACnC,GAAG,QAAQ,0BAA0B,CAAC;IACtC,MAAM;GACR;GACA,IACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;IACA,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C;GACF;GACA,SAAS,CAACC,gBAA0B,OAAO,CAAC,CAAC;GAC7C,IAAI,WACF;GAEF,GAAG,QAAQ,wDAAwD,CAAC;GACpE,YAAY;GACZ;EACF;CAEJ,UAAU;EACR,GAAG,QAAQ,gCAAgC,KAAK,IAAI,IAAI,KAAK;CAC/D;CACA,OAAO;AACT;AAEA,eAAe,sBACb,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,CAAC;CAE3C,eAAe,QAAQ,MAA6C;EAClE,IAAI;GACF,OAAO,MAAM,KAAK,QAAQ;EAC5B,UAAU;GACR,MAAM,IAAI;GAGV,GAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,EAAE,WAAW,OAAO,SAAS;EAC1D;CACF;CAEA,WAAW,kBAAkB;CAE7B,IAAI,CAAC,WAAW;EAGd,MAAM,MAAM,WAAW,YAAY,SAAS,KAAK,EAAE,CAAC,GAAG;EAEvD,MAAM,gBAAgB,WAAW,aAAa,GAAG;EACjD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,aAAa,GACjD,WAAW,gBAAgB,UAAU,aAAa,CACpD,CAAC;EACD,IAAI,UAAU,SACZ,KAAK,MAAM,MAAM,KACf,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,SACE,YAAY,EAAE;EAClB;CAGN;CAIA,MAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,EACX,CACF;CAGA,MAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,KAAK,CAAC,CAAC;AAC7C;AAEA,SAAS,aACP,IACA,QACuC;CACvC,OAAO,EAAE;kBACO,GAAG,OAAO,SAAS,EAAE,GAAG,GAAG,OAAO,KAAK,EAAE;mBACxC,GAAG,OAAO,UAAU,EAAE;;AAEzC;AAEA,SAAS,aACP,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CACvC,OAAO,EAAE;kBACO,GAAG,SAAS,EAAE,GAAG,GAAG,KAAK,EAAE;mBAC1B,GAAG,UAAU,EAAE;oBACd,GAAG,KAAK,EAAE;;AAE9B;AAEA,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,CAAC;CAC7C,KAAK,MAAM,OAAO,YAChB,GAAG,OAAOC,MAAQ,MAAM,MAAM,qBAAqB;CAErD,OAAO,EAAE,UAAU,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,SAAS,OAAO,QAC5D,EACF,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAC/D;AACF;AAEA,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,CAAC;CACpB,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI,WAAW,SAAS,GACtB,WAAW,KAAK,EAAE,KAAK;EAEzB,WAAW,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,MAAM,MAAM;CAChD;CAEA,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,SAAS;AACjD;AAEA,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,KAAK,CAAC,EAAE;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;CAOnD,IAAI,qBAAqB,gBACvB,MAAM,IAAI,8BACR,UACA,oBACA,cACF;MACK,IAAI,qBAAqB,gBAC9B,MAAM,IAAI,cAAc;EACtB,MAAMC;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,SAAS,EAAE;EAC9I,QAAQC;CACV,CAAC;AAEL"}
+{"version":3,"file":"mutagen.js","names":["ErrorKind.MutationRateLimited","MutationType.CRUD","Mode.SERIALIZABLE","ErrorKind.MutationFailed","v.parse","ErrorKind.InvalidPush","ErrorOrigin.ZeroCache"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {resolver} from '../../../../shared/src/resolver.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  type CRUDMutation,\n  type DeleteOp,\n  type InsertOp,\n  type Mutation,\n  type UpdateOp,\n  type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n  primaryKeyValueSchema,\n  type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  WriteAuthorizerImpl,\n  type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n  kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n  desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n  /**\n   * Callers must hold a {@link ref} for the duration of a push (i.e. across\n   * all of its mutations). The service stops, closing its replica handle,\n   * when its ref count drops to zero.\n   */\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled: boolean,\n  ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #upstream: PostgresDB;\n  readonly #shard: ShardID;\n  readonly #stopped = resolver();\n  readonly #replica: Database;\n  readonly #writeAuthorizer: WriteAuthorizerImpl;\n  readonly #limiter: SlidingWindowLimiter | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  readonly #crudMutations = getOrCreateCounter(\n    'mutation',\n    'crud',\n    'Number of CRUD mutations processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    clientGroupID: string,\n    upstream: PostgresDB,\n    config: ZeroConfig,\n    writeAuthzStorage: DatabaseStorage,\n  ) {\n    this.id = clientGroupID;\n    this.#lc = lc;\n    this.#upstream = upstream;\n    this.#shard = shard;\n    this.#replica = new Database(this.#lc, config.replica.file, {\n      fileMustExist: true,\n    });\n    this.#writeAuthorizer = new WriteAuthorizerImpl(\n      this.#lc,\n      config,\n      this.#replica,\n      shard.appID,\n      clientGroupID,\n      writeAuthzStorage,\n    );\n\n    if (config.perUserMutationLimit.max !== undefined) {\n      this.#limiter = new SlidingWindowLimiter(\n        config.perUserMutationLimit.windowMs,\n        config.perUserMutationLimit.max,\n      );\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled = false,\n  ): Promise<MutationError | undefined> {\n    if (this.#limiter?.canDo() === false) {\n      return Promise.resolve([\n        ErrorKind.MutationRateLimited,\n        'Rate limit exceeded',\n      ]);\n    }\n    this.#crudMutations.add(1, {\n      clientGroupID: this.id,\n    });\n    return processMutation(\n      this.#lc,\n      authData,\n      this.#upstream,\n      this.#shard,\n      this.id,\n      mutation,\n      this.#writeAuthorizer,\n      undefined,\n      customMutatorsEnabled,\n    );\n  }\n\n  run(): Promise<void> {\n    return this.#stopped.promise;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return this.#stopped.promise;\n    }\n    this.#isStopped = true;\n    try {\n      this.#writeAuthorizer.destroy();\n    } catch (e) {\n      this.#lc.error?.('error destroying write authorizer storage', e);\n    } finally {\n      // The replica's statistics are maintained by the replicator, so skip\n      // the `PRAGMA optimize` that closing a writable handle would otherwise\n      // run on every client group churn.\n      this.#replica.close({optimize: false});\n    }\n    this.#stopped.resolve();\n    return this.#stopped.promise;\n  }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n  lc: LogContext,\n  authData: JWTPayload | undefined,\n  db: PostgresDB,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: Mutation,\n  writeAuthorizer: WriteAuthorizer,\n  onTxStart?: () => void | Promise<void>, // for testing\n  customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n  assert(\n    mutation.type === MutationType.CRUD,\n    'Only CRUD mutations are supported',\n  );\n  // log-leak-ignore -- the mutation id, not row data\n  lc = lc.withContext('mutationID', mutation.id);\n  lc = lc.withContext('processMutation');\n  // The mutationID context set above identifies it; the mutation itself\n  // carries the row values being written.\n  lc.debug?.('Process mutation start');\n\n  // Record mutation processing attempt for telemetry (regardless of success/failure)\n  recordMutation('crud');\n\n  let result: MutationError | undefined;\n\n  const start = Date.now();\n  try {\n    // Mutations can fail for a variety of reasons:\n    //\n    // - application error\n    // - network/db error\n    // - zero bug\n    //\n    // For application errors what we want is to re-run the mutation in\n    // \"error mode\", which skips the actual mutation and just updates the\n    // lastMutationID. Then return the error to the app.\n    //\n    // However, it's hard to tell the difference between application errors\n    // and the other types.\n    //\n    // A reasonable policy ends up being to just retry every mutation once\n    // in error mode. If the error mode mutation succeeds then we assume it\n    // was an application error and return the error to the app. Otherwise,\n    // we know it was something internal and we log it.\n    //\n    // This is not 100% correct - there are theoretical cases where we\n    // return an internal error to the app that shouldn't have been. But it\n    // would have to be a crazy coincidence: we'd have to have a network\n    // error on the first attempt that resolves by the second attempt.\n    //\n    // One might ask why not try/catch just the calls to the mutators and\n    // consider those application errors. That is actually what we do in\n    // Replicache:\n    //\n    // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n    //\n    // We don't do it here because:\n    //\n    // 1. It's still not perfect. It's hard to isolate SQL errors in\n    //    mutators due to app developer mistakes from SQL errors due to\n    //    Zero mistakes.\n    // 2. It's not possible to do this with the pg library we're using in\n    //    Zero anyway: https://github.com/porsager/postgres/issues/455.\n    //\n    // Personally I think this simple retry policy is nice.\n    let errorMode = false;\n    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n      try {\n        await runTx(\n          db,\n          async tx => {\n            // Simulates a concurrent request for testing. In production this is a noop.\n            const done = onTxStart?.();\n            try {\n              return await processMutationWithTx(\n                lc,\n                tx,\n                authData,\n                shard,\n                clientGroupID,\n                mutation,\n                errorMode,\n                writeAuthorizer,\n              );\n            } finally {\n              await done;\n            }\n          },\n          {mode: Mode.SERIALIZABLE},\n        );\n        if (errorMode) {\n          lc.debug?.('Ran mutation successfully in error mode');\n        }\n        break;\n      } catch (e) {\n        if (e instanceof MutationAlreadyProcessedError) {\n          lc.debug?.(\n            // log-leak-ignore -- mutation and client ids, not row data\n            `${e.message}. Expected mutation ID ${e.lastMutationID}`,\n          );\n          // Don't double-count already processed mutations, but they were counted above\n          return undefined;\n        }\n        if (\n          isProtocolError(e) &&\n          !errorMode &&\n          e.kind === ErrorKind.InvalidPush &&\n          customMutatorsEnabled &&\n          i < 2\n        ) {\n          // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n          // This can create a lot of OOO mutation errors since we do not know when the API server\n          // has applied a custom mutation before moving on to process CRUD mutations.\n          // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n          // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n          // They should migrate completely to custom mutators.\n          lc.info?.(\n            'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n            e,\n          );\n          await new Promise(resolve => setTimeout(resolve, 100));\n          continue;\n        }\n        if (isProtocolError(e) || errorMode) {\n          lc.error?.('Process mutation error', e);\n          throw e;\n        }\n        if (\n          e instanceof postgres.PostgresError &&\n          e.code === PG_SERIALIZATION_FAILURE\n        ) {\n          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n          continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n        }\n        result = [ErrorKind.MutationFailed, String(e)];\n        if (errorMode) {\n          break;\n        }\n        lc.error?.('Got error running mutation, re-running in error mode', e);\n        errorMode = true;\n        i--;\n      }\n    }\n  } finally {\n    lc.debug?.('Process mutation complete in', Date.now() - start);\n  }\n  return result;\n}\n\nasync function processMutationWithTx(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  authData: JWTPayload | undefined,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: CRUDMutation,\n  errorMode: boolean,\n  authorizer: WriteAuthorizer,\n) {\n  const tasks: (() => Promise<unknown>)[] = [];\n\n  async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n    try {\n      return await stmt.execute();\n    } finally {\n      const q = stmt as unknown as Query;\n      // `q.parameters` are the bound row values -- log the parameterized\n      // statement and the parameter count, never the values themselves.\n      lc.debug?.(`${q.string} (${q.parameters.length} params)`);\n    }\n  }\n\n  authorizer.reloadPermissions();\n\n  if (!errorMode) {\n    // Validates table names and keys every op by the replica's primary key,\n    // which is what the authorizer checks.\n    const ops = authorizer.validateOps(mutation.args[0].ops);\n\n    const normalizedOps = authorizer.normalizeOps(ops);\n    const [canPre, canPost] = await Promise.all([\n      authorizer.canPreMutation(authData, normalizedOps),\n      authorizer.canPostMutation(authData, normalizedOps),\n    ]);\n    if (canPre && canPost) {\n      for (const op of ops) {\n        switch (op.op) {\n          case 'insert':\n            tasks.push(() => execute(getInsertSQL(tx, op)));\n            break;\n          case 'upsert':\n            tasks.push(() => execute(getUpsertSQL(tx, op)));\n            break;\n          case 'update':\n            tasks.push(() => execute(getUpdateSQL(tx, op)));\n            break;\n          case 'delete':\n            tasks.push(() => execute(getDeleteSQL(tx, op)));\n            break;\n          default:\n            unreachable(op);\n        }\n      }\n    }\n  }\n\n  // Confirm the mutation even though it may have been blocked by the authorizer.\n  // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n  tasks.unshift(() =>\n    checkSchemaVersionAndIncrementLastMutationID(\n      tx,\n      shard,\n      clientGroupID,\n      mutation.clientID,\n      mutation.id,\n    ),\n  );\n\n  // Note: An error thrown from any Promise aborts the entire transaction.\n  await Promise.all(tasks.map(task => task()));\n}\n\nfunction getInsertSQL(\n  tx: postgres.TransactionSql,\n  create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  return tx`\n    INSERT INTO ${tx(create.tableName)} ${tx(create.value)}\n    ON CONFLICT (${tx(create.primaryKey)}) DO NOTHING\n  `;\n}\n\nfunction getUpsertSQL(\n  tx: postgres.TransactionSql,\n  set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = set;\n  return tx`\n    INSERT INTO ${tx(tableName)} ${tx(value)}\n    ON CONFLICT (${tx(primaryKey)})\n    DO UPDATE SET ${tx(value)}\n  `;\n}\n\nfunction getUpdateSQL(\n  tx: postgres.TransactionSql,\n  update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const table = update.tableName;\n  const {primaryKey, value} = update;\n  const id: Record<string, PrimaryKeyValue> = {};\n  for (const key of primaryKey) {\n    id[key] = v.parse(value[key], primaryKeyValueSchema);\n  }\n  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n    id,\n  ).flatMap(([key, value], i) =>\n    i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n  )}`;\n}\n\nfunction getDeleteSQL(\n  tx: postgres.TransactionSql,\n  deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = deleteOp;\n\n  const conditions = [];\n  for (const key of primaryKey) {\n    if (conditions.length > 0) {\n      conditions.push(tx`AND`);\n    }\n    conditions.push(tx`${tx(key)} = ${value[key]}`);\n  }\n\n  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n  tx: PostgresTransaction,\n  shard: ShardID,\n  clientGroupID: string,\n  clientID: string,\n  receivedMutationID: number,\n) {\n  const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n    INSERT INTO ${tx(upstreamSchema(shard))}.clients \n      as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n          VALUES (${clientGroupID}, ${clientID}, ${1})\n      ON CONFLICT (\"clientGroupID\", \"clientID\")\n      DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n      RETURNING \"lastMutationID\"\n  `;\n\n  // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n  if (receivedMutationID < lastMutationID) {\n    throw new MutationAlreadyProcessedError(\n      clientID,\n      receivedMutationID,\n      lastMutationID,\n    );\n  } else if (receivedMutationID > lastMutationID) {\n    throw new ProtocolError({\n      kind: ErrorKind.InvalidPush,\n      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n      origin: ErrorOrigin.ZeroCache,\n    });\n  }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,oCACF;CAEA,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;EACA,KAAK,KAAK;EACV,KAAK,MAAM;EACX,KAAK,YAAY;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,SAAS,KAAK,KAAK,OAAO,QAAQ,MAAM,EAC1D,eAAe,KACjB,CAAC;EACD,KAAK,mBAAmB,IAAI,oBAC1B,KAAK,KACL,QACA,KAAK,UACL,MAAM,OACN,eACA,iBACF;EAEA,IAAI,OAAO,qBAAqB,QAAQ,KAAA,GACtC,KAAK,WAAW,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,GAC9B;CAEJ;CAEA,MAAM;EACJ,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;EACP,IAAI,KAAK,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAK,YAAY;CAC1B;CAEA,gBACE,UACA,UACA,wBAAwB,OACY;EACpC,IAAI,KAAK,UAAU,MAAM,MAAM,OAC7B,OAAO,QAAQ,QAAQ,CACrBA,qBACA,qBACF,CAAC;EAEH,KAAK,eAAe,IAAI,GAAG,EACzB,eAAe,KAAK,GACtB,CAAC;EACD,OAAO,gBACL,KAAK,KACL,UACA,KAAK,WACL,KAAK,QACL,KAAK,IACL,UACA,KAAK,kBACL,KAAA,GACA,qBACF;CACF;CAEA,MAAqB;EACnB,OAAO,KAAK,SAAS;CACvB;CAEA,OAAsB;EACpB,IAAI,KAAK,YACP,OAAO,KAAK,SAAS;EAEvB,KAAK,aAAa;EAClB,IAAI;GACF,KAAK,iBAAiB,QAAQ;EAChC,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,6CAA6C,CAAC;EACjE,UAAU;GAIR,KAAK,SAAS,MAAM,EAAC,UAAU,MAAK,CAAC;EACvC;EACA,KAAK,SAAS,QAAQ;EACtB,OAAO,KAAK,SAAS;CACvB;AACF;AAEA,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;CACpC,OACE,SAAS,SAASC,MAClB,mCACF;CAEA,KAAK,GAAG,YAAY,cAAc,SAAS,EAAE;CAC7C,KAAK,GAAG,YAAY,iBAAiB;CAGrC,GAAG,QAAQ,wBAAwB;CAGnC,eAAe,MAAM;CAErB,IAAI;CAEJ,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EAuCF,IAAI,YAAY;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,KAC9C,IAAI;GACF,MAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,YAAY;IACzB,IAAI;KACF,OAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,eACF;IACF,UAAU;KACR,MAAM;IACR;GACF,GACA,EAAC,MAAMC,aAAiB,CAC1B;GACA,IAAI,WACF,GAAG,QAAQ,yCAAyC;GAEtD;EACF,SAAS,GAAG;GACV,IAAI,aAAa,+BAA+B;IAC9C,GAAG,QAED,GAAG,EAAE,QAAQ,yBAAyB,EAAE,gBAC1C;IAEA;GACF;GACA,IACE,gBAAgB,CAAC,KACjB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;IAOA,GAAG,OACD,kJACA,CACF;IACA,MAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,GAAG,CAAC;IACrD;GACF;GACA,IAAI,gBAAgB,CAAC,KAAK,WAAW;IACnC,GAAG,QAAQ,0BAA0B,CAAC;IACtC,MAAM;GACR;GACA,IACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;IACA,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C;GACF;GACA,SAAS,CAACC,gBAA0B,OAAO,CAAC,CAAC;GAC7C,IAAI,WACF;GAEF,GAAG,QAAQ,wDAAwD,CAAC;GACpE,YAAY;GACZ;EACF;CAEJ,UAAU;EACR,GAAG,QAAQ,gCAAgC,KAAK,IAAI,IAAI,KAAK;CAC/D;CACA,OAAO;AACT;AAEA,eAAe,sBACb,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,CAAC;CAE3C,eAAe,QAAQ,MAA6C;EAClE,IAAI;GACF,OAAO,MAAM,KAAK,QAAQ;EAC5B,UAAU;GACR,MAAM,IAAI;GAGV,GAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,EAAE,WAAW,OAAO,SAAS;EAC1D;CACF;CAEA,WAAW,kBAAkB;CAE7B,IAAI,CAAC,WAAW;EAGd,MAAM,MAAM,WAAW,YAAY,SAAS,KAAK,EAAE,CAAC,GAAG;EAEvD,MAAM,gBAAgB,WAAW,aAAa,GAAG;EACjD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,aAAa,GACjD,WAAW,gBAAgB,UAAU,aAAa,CACpD,CAAC;EACD,IAAI,UAAU,SACZ,KAAK,MAAM,MAAM,KACf,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,SACE,YAAY,EAAE;EAClB;CAGN;CAIA,MAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,EACX,CACF;CAGA,MAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,KAAK,CAAC,CAAC;AAC7C;AAEA,SAAS,aACP,IACA,QACuC;CACvC,OAAO,EAAE;kBACO,GAAG,OAAO,SAAS,EAAE,GAAG,GAAG,OAAO,KAAK,EAAE;mBACxC,GAAG,OAAO,UAAU,EAAE;;AAEzC;AAEA,SAAS,aACP,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CACvC,OAAO,EAAE;kBACO,GAAG,SAAS,EAAE,GAAG,GAAG,KAAK,EAAE;mBAC1B,GAAG,UAAU,EAAE;oBACd,GAAG,KAAK,EAAE;;AAE9B;AAEA,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,CAAC;CAC7C,KAAK,MAAM,OAAO,YAChB,GAAG,OAAOC,MAAQ,MAAM,MAAM,qBAAqB;CAErD,OAAO,EAAE,UAAU,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,SAAS,OAAO,QAC5D,EACF,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAC/D;AACF;AAEA,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,CAAC;CACpB,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI,WAAW,SAAS,GACtB,WAAW,KAAK,EAAE,KAAK;EAEzB,WAAW,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,MAAM,MAAM;CAChD;CAEA,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,SAAS;AACjD;AAEA,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,KAAK,CAAC,EAAE;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;CAOnD,IAAI,qBAAqB,gBACvB,MAAM,IAAI,8BACR,UACA,oBACA,cACF;MACK,IAAI,qBAAqB,gBAC9B,MAAM,IAAI,cAAc;EACtB,MAAMC;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,SAAS,EAAE;EAC9I,QAAQC;CACV,CAAC;AAEL"}

package/out/zero-cache/src/services/mutagen/pusher.js

@@ -351,7 +351,7 @@ var PushWorker = class {
 			this.#connContextManager.validateConnection(entry.connCtx, entry.connCtx.revision, "kind" in response && response.kind === "MutateResponse" && response?.userID !== void 0 ? {
 				kind: "server-validated",
 				validatedUserID: response.userID
-			} : { kind: "client-fallback" });
+			} : { kind: "server-unverified" });
 			return response;
 		} catch (e) {
 			if (isProtocolError(e) && e.errorBody.kind === "PushFailed") {