@rocicorp/zero 1.11.0-canary.3 → 1.11.0-canary.30

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Sized, Source} from '../../types/streams.ts';\nimport {\n  changeStreamDataSchema,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering while replaying stored changes and\n * filtering changes already seen by a subscriber.\n *\n * Only commit watermarks are exposed to subscribers. The JSON string is the\n * canonical serialization shared by storage and live forwarding.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date. Each result contains parsed, validated data and its approximate\n   * serialized transport size.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<SizedDownstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n//   - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n//     the upstream commit timestamp for end-to-end serving lag measurement.\n//     The stream is parsed in 'passthrough' mode, so an older peer ignores the\n//     field, and a newer peer treats its absence (including in changes\n//     replayed from the Change DB) as \"no commit time reported\".\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string;\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n\n  /**\n   * Legacy topology hint retained on the wire. It no longer affects local\n   * routing: the change-streamer writes the SQLite log itself, so no\n   * subscriber's ACK advances its head or releases its catchup barrier.\n   */\n  logsChangeStream: boolean;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\n/** A downstream message and its approximate serialized transport size. */\nexport type SizedDownstream = Sized<Downstream>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<string>>;\n\n  /**\n   * Starts a snapshot reservation to preserve change-log entries while\n   * a soon-to-be-subscriber downloads the current replica backup. Once\n   * the change-streamer knows that a backup is available\n   * (via {@link trackBackupWatermark}), it sends a confirmation\n   * SnapshotMessage and reserves the corresponding entries in the change-log\n   * until the client with that `taskID` subscribes, or if the snapshot\n   * connection terminates.\n   */\n  startSnapshotReservation(taskID: string): Promise<Source<SnapshotMessage>>;\n\n  /**\n   * Informs the change-streamer of the watermark up to which the replica has\n   * been backed up. This serves two purposes:\n   * - It serves as the maximum watermark up to which the change-log can be\n   *   purged. Note that the change-streamer also takes into account the\n   *   positions of its subscribers and snapshot reservations when purging\n   *   changes in the change-log.\n   * - In RMv2, this ACK's the upstream change-source to allow its buffer of\n   *   changes (e.g. the replication_slot) to advance.\n   */\n  trackBackupWatermark(watermark: string): void;\n}\n"],"mappings":";;;;;;;;AAoKA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {PreSerialized, Sized, Source} from '../../types/streams.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n  changeStreamDataSchema,\n  type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {SubscribeDownstream} from './subscribe.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering while replaying stored changes and\n * filtering changes already seen by a subscriber.\n *\n * Only commit watermarks are exposed to subscribers. The JSON string is the\n * canonical serialization shared by storage and live forwarding.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date. Each result contains parsed, validated data and its approximate\n   * serialized transport size.\n   */\n  subscribe(\n    ctx: SubscriberContext,\n  ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n//   - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n//     the upstream commit timestamp for end-to-end serving lag measurement.\n//     The stream is parsed in 'passthrough' mode, so an older peer ignores the\n//     field, and a newer peer treats its absence (including in changes\n//     replayed from the Change DB) as \"no commit time reported\".\n// v7: v1.12\n//   - Merges the separate `/snapshot` and `/changes` connections into a single\n//     bidirectional `/subscribe` connection (see subscribe.ts). The view-syncer\n//     drives the connection with upstream `reserve-snapshot` / `start-subscription`\n//     application messages; the change-streamer replies with a `reserved`\n//     message and then the change stream on the same socket. This guarantees the\n//     replication-manager that reserves the change log is the one that serves the\n//     subscription. The legacy `/snapshot` + `/changes` endpoints remain for\n//     v4-v6 subscribers during rollout.\n//   - Drops the legacy \"initial\" and \"logsChangeStream\" query parameters from the\n//     subscriber context\n// v8:\n//   - Adds `backfills` to the `start-subscription` context, with which the\n//     subscriber reports its pending backfills (and their progress).\n//   - Adds `progressMarks` to `backfill` and `backfill-completed` messages,\n//     which subscribers use to ignore backfill data that would otherwise\n//     create a gap (see `acceptBackfill()`).\n//   Both fields are optional in the schemas so that v7 peers (and changes\n//   persisted in the change-log by v7 replication-managers) remain compatible,\n//   but they should be considered non-optional as of v8.\n\nexport const PROTOCOL_VERSION = 8;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string;\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether the subscriber supports batched WebSocket frames.\n   */\n  wsBatched?: boolean | undefined;\n\n  /**\n   * The subscriber's pending backfills as of its `watermark`. Reported by\n   * subscribers as of protocol v8. `undefined` for older subscribers.\n   */\n  backfills?: BackfillRequest[] | undefined;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\n/** A downstream message and its approximate serialized transport size. */\nexport type SizedDownstream = Sized<Downstream>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads or pre-serialized Buffers.\n   *\n   * If a `downstream` subscription is supplied it is used directly as the\n   * returned Source. Otherwise, a new one is created.\n   */\n  subscribe(\n    ctx: SubscriberContext,\n    downstream?: Subscription<string | PreSerializedBatch>,\n  ): Promise<Source<string | PreSerialized>>;\n\n  /**\n   * Starts a snapshot reservation to preserve change-log entries while\n   * a soon-to-be-subscriber downloads the current replica backup. Once\n   * the change-streamer knows that a backup is available\n   * (via {@link trackBackupWatermark}), it sends a confirmation\n   * SnapshotMessage and reserves the corresponding entries in the change-log\n   * until the client with that `taskID` subscribes, or if the snapshot\n   * connection terminates.\n   */\n  startSnapshotReservation(taskID: string): Promise<Source<SnapshotMessage>>;\n\n  /**\n   * Informs the change-streamer of the watermark up to which the replica has\n   * been backed up. This serves two purposes:\n   * - It serves as the maximum watermark up to which the change-log can be\n   *   purged. Note that the change-streamer also takes into account the\n   *   positions of its subscribers and snapshot reservations when purging\n   *   changes in the change-log.\n   * - In RMv2, this ACK's the upstream change-source to allow its buffer of\n   *   changes (e.g. the replication_slot) to advance.\n   */\n  trackBackupWatermark(watermark: string): void;\n}\n"],"mappings":";;;;;;;;AA0LA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts

@@ -33,9 +33,9 @@ export declare class Forwarder {
      * two components have an equivalent interpretation of whether a Transaction is
      * currently being streamed.
      *
-     * This version of forward is fire-and-forget, with no flow control. The
-     * change-streamer should call and await {@link forwardWithFlowControl()}
-     * occasionally to avoid memory blowup.
+     * This version of forward buffers changes across the current event loop turn
+     * and dispatches them in batches. The change-streamer should call and await
+     * {@link forwardWithFlowControl()} occasionally to avoid memory blowup.
      */
     forward(entry: WatermarkedChange): void;
     sendStatus(status: Status): void;

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAY,MAAM,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE/E,OAAO,KAAK,EAAC,UAAU,EAAkB,MAAM,iBAAiB,CAAC;AAEjE,MAAM,MAAM,sBAAsB,GAAG;IACnC,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5D,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IAC7C,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF,qBAAa,SAAS;;IA+BpB,YACE,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAEL,EA6DF;IAED,oBAAoB,SAMnB;IAED;;;;;;;OAOG;IACH,uBAAuB,CAAC,GAAG,EAAE,MAAM,QA6DlC;IAED,mBAAmB,SAElB;IAED;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU,QAMlB;IAED,MAAM,CAAC,GAAG,EAAE,UAAU,QAIrB;IAED;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB,QAG/B;IAED,UAAU,CAAC,MAAM,EAAE,MAAM,QAIxB;IAED;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB,iBAyBpD;IA6CD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,CAOrB;CACF"}
+{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAY,MAAM,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE/E,OAAO,KAAK,EAAC,UAAU,EAAkB,MAAM,iBAAiB,CAAC;AAEjE,MAAM,MAAM,sBAAsB,GAAG;IACnC,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5D,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IAC7C,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAIF,qBAAa,SAAS;;IAmCpB,YACE,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAEL,EA6DF;IAED,oBAAoB,SAMnB;IAED;;;;;;;OAOG;IACH,uBAAuB,CAAC,GAAG,EAAE,MAAM,QA6DlC;IAED,mBAAmB,SAGlB;IAED;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU,QAOlB;IAED,MAAM,CAAC,GAAG,EAAE,UAAU,QAKrB;IAED;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB,QAmB/B;IAED,UAAU,CAAC,MAAM,EAAE,MAAM,QAKxB;IAED;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB,iBA4BpD;IAkED,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,CAOrB;CACF"}

package/out/zero-cache/src/services/change-streamer/forwarder.js

@@ -2,6 +2,7 @@ import { joinIterables, wrapIterable } from "../../../../shared/src/iterables.js
 import { getOrCreateCounter, getOrCreateGauge, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
 import { Broadcast } from "./broadcast.js";
 //#region ../zero-cache/src/services/change-streamer/forwarder.ts
+var FORWARD_BATCH_SIZE = 64;
 var Forwarder = class {
 	#lc;
 	#setTimeout;
@@ -13,6 +14,9 @@ var Forwarder = class {
 	#flowControlWaits = getOrCreateCounter("replication", "flow_control.waits", "Flow-control checkpoints completed, labeled by release mode.");
 	#flowControlWaitTime = getOrCreateLatencyHistogram("replication", "flow_control.wait_duration", "Time replication waits at flow-control checkpoints.");
 	#inTransaction = false;
+	#pending = [];
+	#flushScheduled = false;
+	#flushImmediateId = void 0;
 	#currentBroadcast;
 	#progressMonitor;
 	#backupLagEventCount = 0;
@@ -79,6 +83,7 @@ var Forwarder = class {
 	}
 	stopProgressMonitor() {
 		clearInterval(this.#progressMonitor);
+		this.#flushPendingWithoutTracking();
 	}
 	/**
 	* `add()` is called in lock step with `Storer.catchup()` so that the
@@ -86,10 +91,12 @@ var Forwarder = class {
 	* currently being streamed.
 	*/
 	add(sub) {
+		this.#flushPendingWithoutTracking();
 		if (this.#inTransaction) this.#queued.add(sub);
 		else this.#active.add(sub);
 	}
 	remove(sub) {
+		this.#flushPendingWithoutTracking();
 		this.#active.delete(sub);
 		this.#queued.delete(sub);
 		sub.close();
@@ -99,15 +106,22 @@ var Forwarder = class {
 	* two components have an equivalent interpretation of whether a Transaction is
 	* currently being streamed.
 	*
-	* This version of forward is fire-and-forget, with no flow control. The
-	* change-streamer should call and await {@link forwardWithFlowControl()}
-	* occasionally to avoid memory blowup.
+	* This version of forward buffers changes across the current event loop turn
+	* and dispatches them in batches. The change-streamer should call and await
+	* {@link forwardWithFlowControl()} occasionally to avoid memory blowup.
 	*/
 	forward(entry) {
-		Broadcast.withoutTracking(this.#active.values(), entry);
+		this.#pending.push(entry);
+		if (this.#queued.size > 0 && (entry[1] === "commit" || entry[1] === "rollback")) this.#flushPendingWithoutTracking();
 		this.#updateActiveSubscribers(entry[1]);
+		if (this.#pending.length >= FORWARD_BATCH_SIZE) this.#flushPendingWithoutTracking();
+		else if (!this.#flushScheduled && this.#pending.length > 0) {
+			this.#flushScheduled = true;
+			this.#flushImmediateId = setImmediate(() => this.#flushPendingWithoutTracking());
+		}
 	}
 	sendStatus(status) {
+		this.#flushPendingWithoutTracking();
 		for (const sub of this.#active.values()) sub.sendStatus(status);
 	}
 	/**
@@ -115,8 +129,10 @@ var Forwarder = class {
 	* Promise that resolves when replication should continue.
 	*/
 	async forwardWithFlowControl(entry) {
+		this.#pending.push(entry);
+		const batch = this.#drainPending();
 		const start = performance.now();
-		const broadcast = new Broadcast(this.#lc, this.#active.values(), entry, this.#earlyReleaseOptions);
+		const broadcast = new Broadcast(this.#lc, this.#active.values(), batch, this.#earlyReleaseOptions);
 		this.#updateActiveSubscribers(entry[1]);
 		this.#currentBroadcast = broadcast;
 		try {
@@ -128,6 +144,21 @@ var Forwarder = class {
 			if (this.#currentBroadcast === broadcast) this.#currentBroadcast = void 0;
 		}
 	}
+	#drainPending() {
+		if (this.#flushImmediateId !== void 0) {
+			clearImmediate(this.#flushImmediateId);
+			this.#flushImmediateId = void 0;
+		}
+		this.#flushScheduled = false;
+		if (this.#pending.length === 0) return [];
+		const batch = this.#pending;
+		this.#pending = [];
+		return batch;
+	}
+	#flushPendingWithoutTracking() {
+		const batch = this.#drainPending();
+		if (batch.length > 0) Broadcast.withoutTracking(this.#active.values(), batch);
+	}
 	#subscriberStats() {
 		let pendingMessages = 0;
 		let backlogMessages = 0;

package/out/zero-cache/src/services/change-streamer/forwarder.js.map

@@ -1 +1 @@
-{"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#backupLagEventCount","#lastBackupLagWarnMs","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusTimeoutProportion: number;\n  flowControlSlowSubscriberGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #earlyReleaseOptions: EarlyReleaseOptions | undefined;\n  readonly #flowControlSlowSubscriberGracePeriodMs: number | undefined;\n\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  // The backup-replicator is never disconnected as a laggard, so its lag\n  // detections are logged (rather than acted upon). Track the running total and\n  // throttle the warning to at most once per grace period to avoid spamming the\n  // log for the duration of a slow catchup.\n  #backupLagEventCount = 0;\n  #lastBackupLagWarnMs: number | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {\n      flowControlConsensusTimeoutProportion: 2.0,\n    },\n  ) {\n    this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    this.#earlyReleaseOptions =\n      opts.flowControlConsensusTimeoutProportion < 0\n        ? undefined\n        : {\n            consensusTimeoutProportion:\n              opts.flowControlConsensusTimeoutProportion,\n            setTimeoutFn: this.#setTimeout,\n            clearTimeoutFn: this.#clearTimeout,\n          };\n    this.#flowControlSlowSubscriberGracePeriodMs =\n      opts.flowControlSlowSubscriberGracePeriodMs;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(\n      () => this.checkSubscriberProgress(performance.now()),\n      1000,\n    );\n  }\n\n  /**\n   * Runs one progress-monitor tick: samples each active subscriber's change\n   * rate, logs slow broadcasts, and disconnects subscribers that have been\n   * continuously lagging for longer than the configured grace period.\n   *\n   * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n   * than private) so tests can drive a single tick with an explicit `now`.\n   */\n  checkSubscriberProgress(now: number) {\n    let timedOut: Map<Subscriber, SubscriberStats> | undefined;\n    let slowestOnTime: SubscriberStats | undefined;\n\n    for (const sub of this.#active) {\n      const stats = sub.sampleProcessRate(now).getStats();\n      if (stats.missedLastTimeout) {\n        (timedOut ??= new Map()).set(sub, stats);\n      } else if (\n        stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n      ) {\n        slowestOnTime = stats;\n      }\n    }\n    this.#currentBroadcast?.logSlowRequests(now);\n\n    if (\n      this.#flowControlSlowSubscriberGracePeriodMs &&\n      timedOut?.size &&\n      slowestOnTime\n    ) {\n      for (const [sub, stats] of timedOut.entries()) {\n        const rate =\n          stats.processRate < slowestOnTime.processRate\n            ? 'lagging'\n            : 'catching-up';\n        const laggingDuration = sub.reportChangeRate(now, rate);\n        if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n          if (sub.mode === 'backup') {\n            // The backup-replicator can only be detected as a laggard during\n            // initial catchup; after that it is required for consensus. If it\n            // triggers laggard detection in the former case, just log and\n            // continue; as disconnecting it would otherwise kill the\n            // change-streamer task altogether.\n            this.#backupLagEventCount++;\n            const grace = this.#flowControlSlowSubscriberGracePeriodMs;\n            if (\n              this.#lastBackupLagWarnMs === undefined ||\n              now - this.#lastBackupLagWarnMs >= grace\n            ) {\n              this.#lastBackupLagWarnMs = now;\n              this.#lc.warn?.(\n                `backup subscriber ${sub.id} has lagged for ${laggingDuration}ms ` +\n                  `(${this.#backupLagEventCount} lag events so far). ` +\n                  `this is only expected during initial catchup`,\n                {stats},\n              );\n            }\n          } else {\n            this.#lc.warn?.(\n              `serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n              {stats},\n            );\n            sub.close(\n              StreamTooFarBehind,\n              `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n            );\n          }\n        }\n      }\n    }\n  }\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward is fire-and-forget, with no flow control. The\n   * change-streamer should call and await {@link forwardWithFlowControl()}\n   * occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    Broadcast.withoutTracking(this.#active.values(), entry);\n    this.#updateActiveSubscribers(entry[1]);\n  }\n\n  sendStatus(status: Status) {\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    const start = performance.now();\n    const broadcast = new Broadcast(\n      this.#lc,\n      this.#active.values(),\n      entry,\n      this.#earlyReleaseOptions,\n    );\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAoBA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB;CACA;CAMA,uBAAuB;CACvB;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAC1B,IAAI,IAAI,SAAS,UAAU;IAMzB,KAAKM;IACL,MAAM,QAAQ,KAAKN;IACnB,IACE,KAAKO,yBAAyB,KAAA,KAC9B,MAAM,KAAKA,wBAAwB,OACnC;KACA,KAAKA,uBAAuB;KAC5B,KAAKb,IAAI,OACP,qBAAqB,IAAI,GAAG,kBAAkB,gBAAgB,MACxD,KAAKY,qBAAqB,oEAEhC,EAAC,MAAK,CACR;IACF;GACF,OAAO;IACL,KAAKZ,IAAI,OACP,sBAAsB,IAAI,GAAG,kBAAkB,gBAAgB,oBAC/D,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EAEJ;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKI,gBACP,KAAKV,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKY,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKZ,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKH,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKU,yBAAyB,MAAM,EAAE;EAGtC,KAAKJ,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKR,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKU,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKV,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}
+{"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#backupLagEventCount","#lastBackupLagWarnMs","#flushPendingWithoutTracking","#inTransaction","#pending","#updateActiveSubscribers","#flushScheduled","#flushImmediateId","#drainPending"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n  flowControlConsensusTimeoutProportion: number;\n  flowControlSlowSubscriberGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nconst FORWARD_BATCH_SIZE = 64;\n\nexport class Forwarder {\n  readonly #lc: LogContext;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n  readonly #active = new Set<Subscriber>();\n  readonly #queued = new Set<Subscriber>();\n  readonly #earlyReleaseOptions: EarlyReleaseOptions | undefined;\n  readonly #flowControlSlowSubscriberGracePeriodMs: number | undefined;\n\n  readonly #flowControlWaits = getOrCreateCounter(\n    'replication',\n    'flow_control.waits',\n    'Flow-control checkpoints completed, labeled by release mode.',\n  );\n  readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n    'replication',\n    'flow_control.wait_duration',\n    'Time replication waits at flow-control checkpoints.',\n  );\n  #inTransaction = false;\n\n  #pending: WatermarkedChange[] = [];\n  #flushScheduled = false;\n  #flushImmediateId: NodeJS.Immediate | undefined = undefined;\n\n  #currentBroadcast: Broadcast | undefined;\n  #progressMonitor: NodeJS.Timeout | undefined;\n\n  // The backup-replicator is never disconnected as a laggard, so its lag\n  // detections are logged (rather than acted upon). Track the running total and\n  // throttle the warning to at most once per grace period to avoid spamming the\n  // log for the duration of a slow catchup.\n  #backupLagEventCount = 0;\n  #lastBackupLagWarnMs: number | undefined;\n\n  constructor(\n    lc: LogContext,\n    opts: ProgressMonitorOptions = {\n      flowControlConsensusTimeoutProportion: 2.0,\n    },\n  ) {\n    this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    this.#earlyReleaseOptions =\n      opts.flowControlConsensusTimeoutProportion < 0\n        ? undefined\n        : {\n            consensusTimeoutProportion:\n              opts.flowControlConsensusTimeoutProportion,\n            setTimeoutFn: this.#setTimeout,\n            clearTimeoutFn: this.#clearTimeout,\n          };\n    this.#flowControlSlowSubscriberGracePeriodMs =\n      opts.flowControlSlowSubscriberGracePeriodMs;\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.active_subscribers',\n      'Active change-stream subscribers receiving live changes.',\n    ).addCallback(result => result.observe(this.#active.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.queued_subscribers',\n      'Change-stream subscribers waiting for the current transaction to finish before activation.',\n    ).addCallback(result => result.observe(this.#queued.size));\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.pending_messages',\n      'Downstream change-stream messages not yet acked by subscribers.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().pendingMessages),\n    );\n\n    getOrCreateGauge(\n      'replication',\n      'flow_control.backlog_messages',\n      'Live change-stream messages buffered while subscribers catch up.',\n    ).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogMessages),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n      description:\n        'Live change-stream bytes buffered while subscribers catch up.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().backlogBytes),\n    );\n\n    getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n      description:\n        'Maximum live change-stream bytes buffered by a single subscriber.',\n      unit: 'By',\n    }).addCallback(result =>\n      result.observe(this.#subscriberStats().maxBacklogBytes),\n    );\n  }\n\n  startProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#progressMonitor = setInterval(\n      () => this.checkSubscriberProgress(performance.now()),\n      1000,\n    );\n  }\n\n  /**\n   * Runs one progress-monitor tick: samples each active subscriber's change\n   * rate, logs slow broadcasts, and disconnects subscribers that have been\n   * continuously lagging for longer than the configured grace period.\n   *\n   * Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n   * than private) so tests can drive a single tick with an explicit `now`.\n   */\n  checkSubscriberProgress(now: number) {\n    let timedOut: Map<Subscriber, SubscriberStats> | undefined;\n    let slowestOnTime: SubscriberStats | undefined;\n\n    for (const sub of this.#active) {\n      const stats = sub.sampleProcessRate(now).getStats();\n      if (stats.missedLastTimeout) {\n        (timedOut ??= new Map()).set(sub, stats);\n      } else if (\n        stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n      ) {\n        slowestOnTime = stats;\n      }\n    }\n    this.#currentBroadcast?.logSlowRequests(now);\n\n    if (\n      this.#flowControlSlowSubscriberGracePeriodMs &&\n      timedOut?.size &&\n      slowestOnTime\n    ) {\n      for (const [sub, stats] of timedOut.entries()) {\n        const rate =\n          stats.processRate < slowestOnTime.processRate\n            ? 'lagging'\n            : 'catching-up';\n        const laggingDuration = sub.reportChangeRate(now, rate);\n        if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n          if (sub.mode === 'backup') {\n            // The backup-replicator can only be detected as a laggard during\n            // initial catchup; after that it is required for consensus. If it\n            // triggers laggard detection in the former case, just log and\n            // continue; as disconnecting it would otherwise kill the\n            // change-streamer task altogether.\n            this.#backupLagEventCount++;\n            const grace = this.#flowControlSlowSubscriberGracePeriodMs;\n            if (\n              this.#lastBackupLagWarnMs === undefined ||\n              now - this.#lastBackupLagWarnMs >= grace\n            ) {\n              this.#lastBackupLagWarnMs = now;\n              this.#lc.warn?.(\n                `backup subscriber ${sub.id} has lagged for ${laggingDuration}ms ` +\n                  `(${this.#backupLagEventCount} lag events so far). ` +\n                  `this is only expected during initial catchup`,\n                {stats},\n              );\n            }\n          } else {\n            this.#lc.warn?.(\n              `serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n              {stats},\n            );\n            sub.close(\n              StreamTooFarBehind,\n              `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n            );\n          }\n        }\n      }\n    }\n  }\n\n  stopProgressMonitor() {\n    clearInterval(this.#progressMonitor);\n    this.#flushPendingWithoutTracking();\n  }\n\n  /**\n   * `add()` is called in lock step with `Storer.catchup()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   */\n  add(sub: Subscriber) {\n    this.#flushPendingWithoutTracking();\n    if (this.#inTransaction) {\n      this.#queued.add(sub);\n    } else {\n      this.#active.add(sub);\n    }\n  }\n\n  remove(sub: Subscriber) {\n    this.#flushPendingWithoutTracking();\n    this.#active.delete(sub);\n    this.#queued.delete(sub);\n    sub.close();\n  }\n\n  /**\n   * `forward()` is called in lockstep with `Storer.store()` so that the\n   * two components have an equivalent interpretation of whether a Transaction is\n   * currently being streamed.\n   *\n   * This version of forward buffers changes across the current event loop turn\n   * and dispatches them in batches. The change-streamer should call and await\n   * {@link forwardWithFlowControl()} occasionally to avoid memory blowup.\n   */\n  forward(entry: WatermarkedChange) {\n    this.#pending.push(entry);\n\n    if (\n      this.#queued.size > 0 &&\n      (entry[1] === 'commit' || entry[1] === 'rollback')\n    ) {\n      this.#flushPendingWithoutTracking();\n    }\n    this.#updateActiveSubscribers(entry[1]);\n\n    if (this.#pending.length >= FORWARD_BATCH_SIZE) {\n      this.#flushPendingWithoutTracking();\n    } else if (!this.#flushScheduled && this.#pending.length > 0) {\n      this.#flushScheduled = true;\n      this.#flushImmediateId = setImmediate(() =>\n        this.#flushPendingWithoutTracking(),\n      );\n    }\n  }\n\n  sendStatus(status: Status) {\n    this.#flushPendingWithoutTracking();\n    for (const sub of this.#active.values()) {\n      sub.sendStatus(status);\n    }\n  }\n\n  /**\n   * The flow-control-aware equivalent of {@link forward()}, returning a\n   * Promise that resolves when replication should continue.\n   */\n  async forwardWithFlowControl(entry: WatermarkedChange) {\n    this.#pending.push(entry);\n\n    const batch = this.#drainPending();\n    const start = performance.now();\n    const broadcast = new Broadcast(\n      this.#lc,\n      this.#active.values(),\n      batch,\n      this.#earlyReleaseOptions,\n    );\n    this.#updateActiveSubscribers(entry[1]);\n\n    // set for progress tracking\n    this.#currentBroadcast = broadcast;\n\n    try {\n      await broadcast.done;\n      const attributes = {'release.mode': broadcast.releaseMode};\n      this.#flowControlWaits.add(1, attributes);\n      this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n    } finally {\n      // Technically #currentBroadcast may have changed, so only\n      // unset if it if is still the same.\n      if (this.#currentBroadcast === broadcast) {\n        this.#currentBroadcast = undefined;\n      }\n    }\n  }\n\n  #drainPending(): WatermarkedChange[] {\n    if (this.#flushImmediateId !== undefined) {\n      clearImmediate(this.#flushImmediateId);\n      this.#flushImmediateId = undefined;\n    }\n    this.#flushScheduled = false;\n    if (this.#pending.length === 0) {\n      return [];\n    }\n    const batch = this.#pending;\n    this.#pending = [];\n    return batch;\n  }\n\n  #flushPendingWithoutTracking() {\n    const batch = this.#drainPending();\n    if (batch.length > 0) {\n      Broadcast.withoutTracking(this.#active.values(), batch);\n    }\n  }\n\n  #subscriberStats() {\n    let pendingMessages = 0;\n    let backlogMessages = 0;\n    let backlogBytes = 0;\n    let maxBacklogBytes = 0;\n\n    for (const sub of joinIterables(this.#active, this.#queued)) {\n      const stats = sub.getStats();\n      pendingMessages += stats.pending;\n      backlogMessages += stats.backlog;\n      backlogBytes += stats.backlogBytes;\n      maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n    }\n\n    return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n  }\n\n  #updateActiveSubscribers(tag: ChangeTag) {\n    switch (tag) {\n      case 'begin':\n        // While in a Transaction, all added subscribers are \"queued\" so that no\n        // messages are forwarded to them. This state corresponds to being queued\n        // for catchup in the Storer, which will retrieve historic changes\n        // and call catchup() once the current transaction is committed.\n        this.#inTransaction = true;\n        break;\n      case 'commit':\n      case 'rollback':\n        // Upon commit or rollback, all queued subscribers are transferred to\n        // the active set. This means that they can receive messages starting\n        // from the next transaction.\n        //\n        // Note that if catchup is still in progress (in the Storer), these messages\n        // will be buffered in the backlog until catchup completes.\n        this.#inTransaction = false;\n        for (const sub of this.#queued.values()) {\n          this.#active.add(sub);\n        }\n        this.#queued.clear();\n        break;\n    }\n  }\n\n  getAcks(): Set<string> {\n    return new Set(\n      joinIterables(\n        wrapIterable(this.#active).map(s => s.acked),\n        wrapIterable(this.#queued).map(s => s.acked),\n      ),\n    );\n  }\n}\n"],"mappings":";;;;AAoBA,IAAM,qBAAqB;AAE3B,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB,WAAgC,CAAC;CACjC,kBAAkB;CAClB,oBAAkD,KAAA;CAElD;CACA;CAMA,uBAAuB;CACvB;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAC1B,IAAI,IAAI,SAAS,UAAU;IAMzB,KAAKM;IACL,MAAM,QAAQ,KAAKN;IACnB,IACE,KAAKO,yBAAyB,KAAA,KAC9B,MAAM,KAAKA,wBAAwB,OACnC;KACA,KAAKA,uBAAuB;KAC5B,KAAKb,IAAI,OACP,qBAAqB,IAAI,GAAG,kBAAkB,gBAAgB,MACxD,KAAKY,qBAAqB,oEAEhC,EAAC,MAAK,CACR;IACF;GACF,OAAO;IACL,KAAKZ,IAAI,OACP,sBAAsB,IAAI,GAAG,kBAAkB,gBAAgB,oBAC/D,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EAEJ;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;EACnC,KAAKI,6BAA6B;CACpC;;;;;;CAOA,IAAI,KAAiB;EACnB,KAAKA,6BAA6B;EAClC,IAAI,KAAKC,gBACP,KAAKX,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKW,6BAA6B;EAClC,KAAKX,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,KAAKY,SAAS,KAAK,KAAK;EAExB,IACE,KAAKZ,QAAQ,OAAO,MACnB,MAAM,OAAO,YAAY,MAAM,OAAO,aAEvC,KAAKU,6BAA6B;EAEpC,KAAKG,yBAAyB,MAAM,EAAE;EAEtC,IAAI,KAAKD,SAAS,UAAU,oBAC1B,KAAKF,6BAA6B;OAC7B,IAAI,CAAC,KAAKI,mBAAmB,KAAKF,SAAS,SAAS,GAAG;GAC5D,KAAKE,kBAAkB;GACvB,KAAKC,oBAAoB,mBACvB,KAAKL,6BAA6B,CACpC;EACF;CACF;CAEA,WAAW,QAAgB;EACzB,KAAKA,6BAA6B;EAClC,KAAK,MAAM,OAAO,KAAKX,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,KAAKa,SAAS,KAAK,KAAK;EAExB,MAAM,QAAQ,KAAKI,cAAc;EACjC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKpB,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKY,yBAAyB,MAAM,EAAE;EAGtC,KAAKN,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,gBAAqC;EACnC,IAAI,KAAKQ,sBAAsB,KAAA,GAAW;GACxC,eAAe,KAAKA,iBAAiB;GACrC,KAAKA,oBAAoB,KAAA;EAC3B;EACA,KAAKD,kBAAkB;EACvB,IAAI,KAAKF,SAAS,WAAW,GAC3B,OAAO,CAAC;EAEV,MAAM,QAAQ,KAAKA;EACnB,KAAKA,WAAW,CAAC;EACjB,OAAO;CACT;CAEA,+BAA+B;EAC7B,MAAM,QAAQ,KAAKI,cAAc;EACjC,IAAI,MAAM,SAAS,GACjB,UAAU,gBAAgB,KAAKjB,QAAQ,OAAO,GAAG,KAAK;CAE1D;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKA,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKW,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKX,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.d.ts

@@ -0,0 +1,53 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Source } from '../../types/streams.ts';
+import { LitestreamController } from '../litestream/litestream-controller.ts';
+import type { BackedUpWatermark } from './backup-monitor.ts';
+/** The subset of {@link LitestreamController} used by the requester. */
+export type SyncController = Pick<LitestreamController, 'sync' | 'close'>;
+export type SyncRequesterConfig = {
+    /**
+     * Interval between sync requests. This paces how often litestream seals
+     * (and uploads) a new LTX file.
+     */
+    intervalMs: number;
+};
+/**
+ * The LitestreamSyncRequester drives litestream v5 backups and tracks which
+ * replica watermark is durably backed up, emitting it as a stream of
+ * {@link BackedUpWatermark}s.
+ *
+ * It periodically reads the replica's current watermark `W` and then asks
+ * litestream to sync and wait for the upload (`POST /sync {wait: true}`).
+ * litestream satisfies the request with a sync pass that *starts after* the
+ * request, and a pass syncs up to the end of the WAL, so a successful response
+ * means that every transaction committed before the request, including the
+ * one that wrote `W`, has been uploaded to the backup. No part of the backup
+ * needs to be read to determine this.
+ *
+ * Requests continue even when `W` is already backed up, so that changes that
+ * don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)
+ * are also backed up at this cadence rather than only at litestream's slower
+ * backstop interval. A request on an idle replica is a local no-op for
+ * litestream: there is nothing to seal or upload, and no backup reads.
+ *
+ * At most one request is outstanding at a time. Failed or timed-out requests
+ * are retried with a backoff that starts short and, once it grows to the
+ * interval, defers to the regular ticks. Nothing is published until a request
+ * succeeds.
+ */
+export declare class LitestreamSyncRequester {
+    #private;
+    constructor(lc: LogContext, replicaFile: string, config: SyncRequesterConfig, litestream?: SyncController);
+    /**
+     * Starts requesting syncs and pushes backed-up watermarks to the returned
+     * stream. Must only be called once.
+     */
+    start(): Source<BackedUpWatermark>;
+    /**
+     * Requests a sync unless a request is already outstanding. Resolves when
+     * this tick's request (or the outstanding one) completes.
+     * Exported for testing.
+     */
+    readonly tick: () => Promise<void>;
+}
+//# sourceMappingURL=litestream-sync-requester.d.ts.map

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream-sync-requester.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/litestream-sync-requester.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAEnD,OAAO,EAAC,oBAAoB,EAAC,MAAM,wCAAwC,CAAC;AAE5E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAoC3D,wEAAwE;AACxE,MAAM,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC;AAE1E,MAAM,MAAM,mBAAmB,GAAG;IAChC;;;OAGG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,qBAAa,uBAAuB;;IAelC,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,mBAAmB,EAC3B,UAAU,GAAE,cAA0D,EAmBvE;IAED;;;OAGG;IACH,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC,CAQjC;IAED;;;;OAIG;IACH,QAAQ,CAAC,IAAI,QAAO,OAAO,CAAC,IAAI,CAAC,CAe/B;CAqDH"}

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.js

@@ -0,0 +1,156 @@
+import { assert } from "../../../../shared/src/asserts.js";
+import { parse, valita_exports } from "../../../../shared/src/valita.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { RunningState } from "../running-state.js";
+import { Subscription } from "../../types/subscription.js";
+import { LitestreamController } from "../litestream/litestream-controller.js";
+//#region ../zero-cache/src/services/change-streamer/litestream-sync-requester.ts
+/**
+* How long litestream waits for a requested sync (and its upload) before
+* ending the request. A request that times out is simply retried on a later
+* tick: litestream only abandons the *wait*, never the sync or upload itself,
+* so this bounds request lifetime without affecting backup progress.
+*/
+var SERVER_TIMEOUT_SECONDS = 60;
+/**
+* Client-side timeout, slightly above the server timeout so that litestream,
+* rather than the client, ends a slow request.
+*/
+var CLIENT_TIMEOUT_MS = (SERVER_TIMEOUT_SECONDS + 5) * 1e3;
+var ERROR_LOG_INTERVAL_MS = 6e4;
+/**
+* Initial delay before retrying a failed request, doubled on each consecutive
+* failure until it reaches the configured interval, after which the regular
+* ticks alone retry, so that a busy litestream sees no more requests than the
+* interval allows. The first request is typically issued before litestream's
+* control socket is listening; retrying quickly (rather than after a full
+* interval) keeps the initial backup confirmation, which gates readiness, from
+* waiting out an interval.
+*/
+var MIN_RETRY_DELAY_MS = 250;
+var localWatermarkSchema = valita_exports.object({
+	stateVersion: valita_exports.string(),
+	writeTimeMs: valita_exports.number()
+});
+/**
+* The LitestreamSyncRequester drives litestream v5 backups and tracks which
+* replica watermark is durably backed up, emitting it as a stream of
+* {@link BackedUpWatermark}s.
+*
+* It periodically reads the replica's current watermark `W` and then asks
+* litestream to sync and wait for the upload (`POST /sync {wait: true}`).
+* litestream satisfies the request with a sync pass that *starts after* the
+* request, and a pass syncs up to the end of the WAL, so a successful response
+* means that every transaction committed before the request, including the
+* one that wrote `W`, has been uploaded to the backup. No part of the backup
+* needs to be read to determine this.
+*
+* Requests continue even when `W` is already backed up, so that changes that
+* don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)
+* are also backed up at this cadence rather than only at litestream's slower
+* backstop interval. A request on an idle replica is a local no-op for
+* litestream: there is nothing to seal or upload, and no backup reads.
+*
+* At most one request is outstanding at a time. Failed or timed-out requests
+* are retried with a backoff that starts short and, once it grows to the
+* interval, defers to the regular ticks. Nothing is published until a request
+* succeeds.
+*/
+var LitestreamSyncRequester = class {
+	#lc;
+	#state = new RunningState("litestream-sync-requester");
+	#config;
+	#litestream;
+	#readLocalWatermark;
+	#stream;
+	#timer;
+	#retryTimer;
+	#retryDelayMs = MIN_RETRY_DELAY_MS;
+	#inFlight;
+	#backedUp;
+	#lastErrorLogMs = 0;
+	constructor(lc, replicaFile, config, litestream = new LitestreamController(lc, replicaFile)) {
+		this.#lc = lc.withContext("component", "litestream-sync-requester");
+		this.#config = config;
+		this.#litestream = litestream;
+		const db = new Database(this.#lc, replicaFile, { readonly: true });
+		this.#readLocalWatermark = db.prepare(`SELECT stateVersion, writeTimeMs FROM "_zero.replicationState"`);
+		this.#stream = Subscription.create({ cleanup: () => {
+			clearInterval(this.#timer);
+			clearTimeout(this.#retryTimer);
+			this.#state.stop(this.#lc);
+			this.#litestream.close();
+			db.close();
+		} });
+	}
+	/**
+	* Starts requesting syncs and pushes backed-up watermarks to the returned
+	* stream. Must only be called once.
+	*/
+	start() {
+		assert(this.#timer === void 0, `Already called start()`);
+		this.#lc.info?.(`requesting litestream syncs every ${this.#config.intervalMs} ms`);
+		this.#timer = setInterval(this.tick, this.#config.intervalMs);
+		this.tick();
+		return this.#stream;
+	}
+	/**
+	* Requests a sync unless a request is already outstanding. Resolves when
+	* this tick's request (or the outstanding one) completes.
+	* Exported for testing.
+	*/
+	tick = () => {
+		if (!this.#state.shouldRun() || this.#inFlight) return this.#inFlight ?? Promise.resolve();
+		let local;
+		try {
+			local = parse(this.#readLocalWatermark.get(), localWatermarkSchema);
+		} catch (e) {
+			this.#lc.warn?.(`unable to read local watermark`, e);
+			return Promise.resolve();
+		}
+		this.#inFlight = this.#requestSync(local).finally(() => {
+			this.#inFlight = void 0;
+		});
+		return this.#inFlight;
+	};
+	async #requestSync(local) {
+		try {
+			const result = await this.#litestream.sync({
+				wait: true,
+				timeoutMs: CLIENT_TIMEOUT_MS,
+				serverTimeoutSeconds: SERVER_TIMEOUT_SECONDS
+			}, this.#state.signal);
+			this.#retryDelayMs = MIN_RETRY_DELAY_MS;
+			this.#publish(local, Date.now());
+			this.#lc.debug?.(`backed up watermark ${local.stateVersion}`, result);
+		} catch (e) {
+			if (!this.#state.shouldRun()) return;
+			this.#scheduleRetry();
+			const now = Date.now();
+			if (now - this.#lastErrorLogMs >= ERROR_LOG_INTERVAL_MS) {
+				this.#lastErrorLogMs = now;
+				this.#lc.info?.(`backup of watermark ${local.stateVersion} not yet confirmed. retrying`, e);
+			}
+		}
+	}
+	#scheduleRetry() {
+		clearTimeout(this.#retryTimer);
+		const delayMs = this.#retryDelayMs;
+		if (delayMs >= this.#config.intervalMs) return;
+		this.#retryDelayMs = delayMs * 2;
+		this.#retryTimer = setTimeout(this.tick, delayMs);
+	}
+	#publish({ stateVersion, writeTimeMs }, backupTimeMs) {
+		if (this.#backedUp !== void 0 && stateVersion <= this.#backedUp) return;
+		this.#backedUp = stateVersion;
+		this.#stream.push({
+			watermark: stateVersion,
+			writeTimeMs,
+			backupTimeMs
+		});
+	}
+};
+//#endregion
+export { LitestreamSyncRequester };
+
+//# sourceMappingURL=litestream-sync-requester.js.map

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream-sync-requester.js","names":["#lc","#state","#config","#litestream","#readLocalWatermark","#stream","#timer","#retryTimer","#inFlight","#requestSync","#retryDelayMs","#publish","#scheduleRetry","#lastErrorLogMs","#backedUp"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream-sync-requester.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database, type Statement} from '../../../../zqlite/src/db.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {LitestreamController} from '../litestream/litestream-controller.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\n/**\n * How long litestream waits for a requested sync (and its upload) before\n * ending the request. A request that times out is simply retried on a later\n * tick: litestream only abandons the *wait*, never the sync or upload itself,\n * so this bounds request lifetime without affecting backup progress.\n */\nconst SERVER_TIMEOUT_SECONDS = 60;\n\n/**\n * Client-side timeout, slightly above the server timeout so that litestream,\n * rather than the client, ends a slow request.\n */\nconst CLIENT_TIMEOUT_MS = (SERVER_TIMEOUT_SECONDS + 5) * 1000;\n\nconst ERROR_LOG_INTERVAL_MS = 60_000;\n\n/**\n * Initial delay before retrying a failed request, doubled on each consecutive\n * failure until it reaches the configured interval, after which the regular\n * ticks alone retry, so that a busy litestream sees no more requests than the\n * interval allows. The first request is typically issued before litestream's\n * control socket is listening; retrying quickly (rather than after a full\n * interval) keeps the initial backup confirmation, which gates readiness, from\n * waiting out an interval.\n */\nconst MIN_RETRY_DELAY_MS = 250;\n\nconst localWatermarkSchema = v.object({\n  stateVersion: v.string(),\n  writeTimeMs: v.number(),\n});\n\ntype LocalWatermark = v.Infer<typeof localWatermarkSchema>;\n\n/** The subset of {@link LitestreamController} used by the requester. */\nexport type SyncController = Pick<LitestreamController, 'sync' | 'close'>;\n\nexport type SyncRequesterConfig = {\n  /**\n   * Interval between sync requests. This paces how often litestream seals\n   * (and uploads) a new LTX file.\n   */\n  intervalMs: number;\n};\n\n/**\n * The LitestreamSyncRequester drives litestream v5 backups and tracks which\n * replica watermark is durably backed up, emitting it as a stream of\n * {@link BackedUpWatermark}s.\n *\n * It periodically reads the replica's current watermark `W` and then asks\n * litestream to sync and wait for the upload (`POST /sync {wait: true}`).\n * litestream satisfies the request with a sync pass that *starts after* the\n * request, and a pass syncs up to the end of the WAL, so a successful response\n * means that every transaction committed before the request, including the\n * one that wrote `W`, has been uploaded to the backup. No part of the backup\n * needs to be read to determine this.\n *\n * Requests continue even when `W` is already backed up, so that changes that\n * don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)\n * are also backed up at this cadence rather than only at litestream's slower\n * backstop interval. A request on an idle replica is a local no-op for\n * litestream: there is nothing to seal or upload, and no backup reads.\n *\n * At most one request is outstanding at a time. Failed or timed-out requests\n * are retried with a backoff that starts short and, once it grows to the\n * interval, defers to the regular ticks. Nothing is published until a request\n * succeeds.\n */\nexport class LitestreamSyncRequester {\n  readonly #lc: LogContext;\n  readonly #state = new RunningState('litestream-sync-requester');\n  readonly #config: SyncRequesterConfig;\n  readonly #litestream: SyncController;\n  readonly #readLocalWatermark: Statement;\n  readonly #stream: Subscription<BackedUpWatermark>;\n\n  #timer: NodeJS.Timeout | undefined;\n  #retryTimer: NodeJS.Timeout | undefined;\n  #retryDelayMs = MIN_RETRY_DELAY_MS;\n  #inFlight: Promise<void> | undefined;\n  #backedUp: string | undefined;\n  #lastErrorLogMs = 0;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    config: SyncRequesterConfig,\n    litestream: SyncController = new LitestreamController(lc, replicaFile),\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-sync-requester');\n    this.#config = config;\n    this.#litestream = litestream;\n\n    const db = new Database(this.#lc, replicaFile, {readonly: true});\n    this.#readLocalWatermark = db.prepare(\n      `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n    );\n    this.#stream = Subscription.create<BackedUpWatermark>({\n      cleanup: () => {\n        clearInterval(this.#timer);\n        clearTimeout(this.#retryTimer);\n        this.#state.stop(this.#lc);\n        this.#litestream.close();\n        db.close();\n      },\n    });\n  }\n\n  /**\n   * Starts requesting syncs and pushes backed-up watermarks to the returned\n   * stream. Must only be called once.\n   */\n  start(): Source<BackedUpWatermark> {\n    assert(this.#timer === undefined, `Already called start()`);\n    this.#lc.info?.(\n      `requesting litestream syncs every ${this.#config.intervalMs} ms`,\n    );\n    this.#timer = setInterval(this.tick, this.#config.intervalMs);\n    void this.tick();\n    return this.#stream;\n  }\n\n  /**\n   * Requests a sync unless a request is already outstanding. Resolves when\n   * this tick's request (or the outstanding one) completes.\n   * Exported for testing.\n   */\n  readonly tick = (): Promise<void> => {\n    if (!this.#state.shouldRun() || this.#inFlight) {\n      return this.#inFlight ?? Promise.resolve();\n    }\n    let local: LocalWatermark;\n    try {\n      local = v.parse(this.#readLocalWatermark.get(), localWatermarkSchema);\n    } catch (e) {\n      this.#lc.warn?.(`unable to read local watermark`, e);\n      return Promise.resolve();\n    }\n    this.#inFlight = this.#requestSync(local).finally(() => {\n      this.#inFlight = undefined;\n    });\n    return this.#inFlight;\n  };\n\n  async #requestSync(local: LocalWatermark) {\n    // `local` must be read before the request is issued: litestream only\n    // guarantees that transactions committed before the request are synced.\n    try {\n      const result = await this.#litestream.sync(\n        {\n          wait: true,\n          timeoutMs: CLIENT_TIMEOUT_MS,\n          serverTimeoutSeconds: SERVER_TIMEOUT_SECONDS,\n        },\n        this.#state.signal,\n      );\n      this.#retryDelayMs = MIN_RETRY_DELAY_MS;\n      this.#publish(local, Date.now());\n      this.#lc.debug?.(`backed up watermark ${local.stateVersion}`, result);\n    } catch (e) {\n      if (!this.#state.shouldRun()) {\n        return; // shutting down\n      }\n      // Expected while litestream is starting up or busy with a long sync or\n      // upload (e.g. the initial snapshot of a large replica); the request is\n      // retried after a backoff, or on the next tick if that comes first.\n      this.#scheduleRetry();\n      const now = Date.now();\n      if (now - this.#lastErrorLogMs >= ERROR_LOG_INTERVAL_MS) {\n        this.#lastErrorLogMs = now;\n        this.#lc.info?.(\n          `backup of watermark ${local.stateVersion} not yet confirmed. retrying`,\n          e,\n        );\n      }\n    }\n  }\n\n  #scheduleRetry() {\n    clearTimeout(this.#retryTimer);\n    const delayMs = this.#retryDelayMs;\n    if (delayMs >= this.#config.intervalMs) {\n      return; // backed off to the interval; the regular ticks retry\n    }\n    this.#retryDelayMs = delayMs * 2;\n    this.#retryTimer = setTimeout(this.tick, delayMs);\n  }\n\n  #publish({stateVersion, writeTimeMs}: LocalWatermark, backupTimeMs: number) {\n    if (this.#backedUp !== undefined && stateVersion <= this.#backedUp) {\n      return;\n    }\n    this.#backedUp = stateVersion;\n    this.#stream.push({watermark: stateVersion, writeTimeMs, backupTimeMs});\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AAgBA,IAAM,yBAAyB;;;;;AAM/B,IAAM,qBAAqB,yBAAyB,KAAK;AAEzD,IAAM,wBAAwB;;;;;;;;;;AAW9B,IAAM,qBAAqB;AAE3B,IAAM,uBAAuB,eAAE,OAAO;CACpC,cAAc,eAAE,OAAO;CACvB,aAAa,eAAE,OAAO;AACxB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;AAuCD,IAAa,0BAAb,MAAqC;CACnC;CACA,SAAkB,IAAI,aAAa,2BAA2B;CAC9D;CACA;CACA;CACA;CAEA;CACA;CACA,gBAAgB;CAChB;CACA;CACA,kBAAkB;CAElB,YACE,IACA,aACA,QACA,aAA6B,IAAI,qBAAqB,IAAI,WAAW,GACrE;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAKE,UAAU;EACf,KAAKC,cAAc;EAEnB,MAAM,KAAK,IAAI,SAAS,KAAKH,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,KAAKI,sBAAsB,GAAG,QAC5B,gEACF;EACA,KAAKC,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAKC,MAAM;GACzB,aAAa,KAAKC,WAAW;GAC7B,KAAKN,OAAO,KAAK,KAAKD,GAAG;GACzB,KAAKG,YAAY,MAAM;GACvB,GAAG,MAAM;EACX,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKG,WAAW,KAAA,GAAW,wBAAwB;EAC1D,KAAKN,IAAI,OACP,qCAAqC,KAAKE,QAAQ,WAAW,IAC/D;EACA,KAAKI,SAAS,YAAY,KAAK,MAAM,KAAKJ,QAAQ,UAAU;EAC5D,KAAU,KAAK;EACf,OAAO,KAAKG;CACd;;;;;;CAOA,aAAqC;EACnC,IAAI,CAAC,KAAKJ,OAAO,UAAU,KAAK,KAAKO,WACnC,OAAO,KAAKA,aAAa,QAAQ,QAAQ;EAE3C,IAAI;EACJ,IAAI;GACF,QAAQ,MAAQ,KAAKJ,oBAAoB,IAAI,GAAG,oBAAoB;EACtE,SAAS,GAAG;GACV,KAAKJ,IAAI,OAAO,kCAAkC,CAAC;GACnD,OAAO,QAAQ,QAAQ;EACzB;EACA,KAAKQ,YAAY,KAAKC,aAAa,KAAK,EAAE,cAAc;GACtD,KAAKD,YAAY,KAAA;EACnB,CAAC;EACD,OAAO,KAAKA;CACd;CAEA,MAAMC,aAAa,OAAuB;EAGxC,IAAI;GACF,MAAM,SAAS,MAAM,KAAKN,YAAY,KACpC;IACE,MAAM;IACN,WAAW;IACX,sBAAsB;GACxB,GACA,KAAKF,OAAO,MACd;GACA,KAAKS,gBAAgB;GACrB,KAAKC,SAAS,OAAO,KAAK,IAAI,CAAC;GAC/B,KAAKX,IAAI,QAAQ,uBAAuB,MAAM,gBAAgB,MAAM;EACtE,SAAS,GAAG;GACV,IAAI,CAAC,KAAKC,OAAO,UAAU,GACzB;GAKF,KAAKW,eAAe;GACpB,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,MAAM,KAAKC,mBAAmB,uBAAuB;IACvD,KAAKA,kBAAkB;IACvB,KAAKb,IAAI,OACP,uBAAuB,MAAM,aAAa,+BAC1C,CACF;GACF;EACF;CACF;CAEA,iBAAiB;EACf,aAAa,KAAKO,WAAW;EAC7B,MAAM,UAAU,KAAKG;EACrB,IAAI,WAAW,KAAKR,QAAQ,YAC1B;EAEF,KAAKQ,gBAAgB,UAAU;EAC/B,KAAKH,cAAc,WAAW,KAAK,MAAM,OAAO;CAClD;CAEA,SAAS,EAAC,cAAc,eAA8B,cAAsB;EAC1E,IAAI,KAAKO,cAAc,KAAA,KAAa,gBAAgB,KAAKA,WACvD;EAEF,KAAKA,YAAY;EACjB,KAAKT,QAAQ,KAAK;GAAC,WAAW;GAAc;GAAa;EAAY,CAAC;CACxE;AACF"}

package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts

@@ -48,7 +48,64 @@ interface PurgeLock {
 }
 export declare function setupCDCTables(lc: LogContext, db: postgres.TransactionSql, shard: ShardID): Promise<void>;
 export declare function markResetRequired(sql: PostgresDB, shard: ShardID): Promise<void>;
-export declare function ensureReplicationConfig(lc: LogContext, db: PostgresDB, subscriptionState: Pick<SubscriptionState, 'publications' | 'replicaVersion' | 'watermark'>, shard: ShardID, autoReset: boolean, purgeLock?: PurgeLock, setTimeoutFn?: typeof setTimeout): Promise<void>;
+export declare function ensureReplicationConfig(lc: LogContext, db: PostgresDB, subscriptionState: Pick<SubscriptionState, 'publications' | 'replicaVersion' | 'watermark'>, shard: ShardID, autoReset: boolean, purgeLock?: PurgeLock, setTimeoutFn?: typeof setTimeout, pgChangeLogEnabled?: boolean): Promise<void>;
+export type ChangeLogReinitialization = {
+    /**
+     * The watermark of the replica from which the change log is re-initialized,
+     * i.e. the watermark from which the change stream will resume.
+     */
+    watermark: string;
+    cookies: {
+        tableMetadata: readonly {
+            schema: string;
+            table: string;
+            metadata: TableMetadata;
+        }[];
+        backfilling: readonly {
+            schema: string;
+            table: string;
+            column: string;
+            backfill: BackfillID;
+        }[];
+    };
+    owner: string;
+    ownerAddress: string;
+};
+/**
+ * Re-initializes the change log from the state of the replica if it does not
+ * contain all of the replica's changes, i.e. if its `lastWatermark` is before
+ * the replica's watermark. This is the case if the change log is no longer
+ * maintained (e.g. after running in high-availability mode), or if its last
+ * owner stopped before persisting changes that it had already forwarded
+ * (and backed up). Ownership of the change log is taken in the same
+ * transaction, in which:
+ *
+ * * The `replicationState`, `changeLog`, `tableMetadata`, and `backfilling`
+ *   tables are truncated. (The `replicationConfig` is kept;
+ *   {@link ensureReplicationConfig} handles changes of the replica version
+ *   or publications.)
+ * * The change log is seeded with an empty transaction at the replica's
+ *   watermark, so that subscribers at that watermark can be caught up, while
+ *   subscribers at earlier watermarks are rejected as being too old.
+ * * The `tableMetadata` and `backfilling` cookies are set from the replica.
+ *
+ * Returns `'maintained'` without modifying anything if the change log's
+ * `lastWatermark` is at or past the replica's watermark, i.e. the change log
+ * contains all of the replica's changes. Because a replication slot is only
+ * acked past a transaction once it has been backed up, this means that the
+ * change log contains every transaction up to the position of the replica's
+ * replication slot, and it can be taken over (and resumed from its head).
+ *
+ * This is called when the change log's head is behind the position of the
+ * replication slot (see `PgChangeLogPurgeLocker`). That position cannot be
+ * compared with the watermarks of transactions to determine whether the
+ * change log is maintained, as it is also advanced past positions of changes
+ * outside of the publication.
+ *
+ * @param purgeLock A purge lock held by the caller, if any, which is released
+ *     before the tables are truncated.
+ */
+export declare function reinitializeChangeLog(lc: LogContext, db: PostgresDB, shard: ShardID, { watermark, cookies, owner, ownerAddress }: ChangeLogReinitialization, purgeLock?: PurgeLock, setTimeoutFn?: typeof setTimeout): Promise<'reinitialized' | 'maintained'>;
 export declare const CHANGE_STREAMER_APP_NAME = "zero-change-streamer";
 export declare class AutoResetSignal extends AbortError {
     readonly name = "AutoResetSignal";

package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EACV,UAAU,EACV,MAAM,EACN,aAAa,EACd,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAOpF,eAAO,MAAM,SAAS,QAAQ,CAAC;AAM/B,MAAM,MAAM,cAAc,GAAG;IAG3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAmBF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,UASzD;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CACjC,CAAC;AAaF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,UAiBlD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,aAAa,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAYF,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,KAAK,EAAE,OAAO,iBAIf;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,iBAKtE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,iBAAiB,EAAE,IAAI,CACrB,iBAAiB,EACjB,cAAc,GAAG,gBAAgB,GAAG,WAAW,CAChD,EACD,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,OAAO,EAClB,SAAS,CAAC,EAAE,SAAS,EACrB,YAAY,GAAE,OAAO,UAAuB,iBA4H7C;AAMD,eAAO,MAAM,wBAAwB,yBAAyB,CAAC;AAE/D,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,qBAAqB;CACnC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,iBA4Cf"}
+{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EACV,UAAU,EACV,MAAM,EACN,aAAa,EACd,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAOpF,eAAO,MAAM,SAAS,QAAQ,CAAC;AAM/B,MAAM,MAAM,cAAc,GAAG;IAG3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAmBF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,UASzD;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CACjC,CAAC;AAaF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,UAiBlD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,aAAa,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAYF,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,KAAK,EAAE,OAAO,iBAIf;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,iBAKtE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,iBAAiB,EAAE,IAAI,CACrB,iBAAiB,EACjB,cAAc,GAAG,gBAAgB,GAAG,WAAW,CAChD,EACD,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,OAAO,EAClB,SAAS,CAAC,EAAE,SAAS,EACrB,YAAY,GAAE,OAAO,UAAuB,EAC5C,kBAAkB,UAAO,iBAyH1B;AA6CD,MAAM,MAAM,yBAAyB,GAAG;IACtC;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QACP,aAAa,EAAE,SAAS;YACtB,MAAM,EAAE,MAAM,CAAC;YACf,KAAK,EAAE,MAAM,CAAC;YACd,QAAQ,EAAE,aAAa,CAAC;SACzB,EAAE,CAAC;QACJ,WAAW,EAAE,SAAS;YACpB,MAAM,EAAE,MAAM,CAAC;YACf,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;YACf,QAAQ,EAAE,UAAU,CAAC;SACtB,EAAE,CAAC;KACL,CAAC;IACF,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAgB,qBAAqB,CACnC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAC,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAC,EAAE,yBAAyB,EACpE,SAAS,CAAC,EAAE,SAAS,EACrB,YAAY,GAAE,OAAO,UAAuB,GAC3C,OAAO,CAAC,eAAe,GAAG,YAAY,CAAC,CAuDzC;AAED,eAAO,MAAM,wBAAwB,yBAAyB,CAAC;AAE/D,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,qBAAqB;CACnC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,iBA4Cf"}