npm - @rocicorp/zero - Versions diffs - 1.11.0-head-86ca5eff-20261014 → 1.11.1 - Mend

@rocicorp/zero 1.11.0-head-86ca5eff-20261014 → 1.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (750) hide show

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {PreSerialized, Sized, Source} from '../../types/streams.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n changeStreamDataSchema,\n type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {LocalReplica, SubscribeDownstream} from './subscribe.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/*\n Internally all downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering while replaying stored changes and\n * filtering changes already seen by a subscriber.\n \n Only commit watermarks are exposed to subscribers. The JSON string is the\n * canonical serialization shared by storage and live forwarding.\n /\nexport type WatermarkedChange = [\n watermark: string,\n tag: ChangeTag,\n json: string,\n];\n\n/\n The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n \n * Changes from the upstream source are immediately forwarded to\n * connected subscribers to minimize latency.\n \n * They are then stored in a separate DB to facilitate catchup\n * of connecting subscribers that are behind.\n \n * Acknowledgements are sent upstream after they are successfully\n * stored.\n \n Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n \n However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n \n The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n /\nexport interface ChangeStreamer {\n /\n Subscribes to changes based on the supplied subscriber `ctx`,\n * which indicates the watermark at which the subscriber is up to\n * date. Each result contains parsed, validated data and its approximate\n * serialized transport size.\n /\n subscribe(\n ctx: SubscriberContext,\n ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\n// v1: v0.18\n// - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n// - Adds the \"status\" message which is initially used to signal that the\n// subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n// - Adds the \"taskID\" to the subscription context, and support for\n// the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n// - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n// status request so that a subscriber can verify whether its replica,\n// whether it be restored or existing in a permanent volume, is compatible\n// with the change-streamer.\n// v5: v0.26\n// - Moves relation.keyColumns and relation.replicaIdentity to\n// relation.rowKey: { columns, type }.\n// - Adds `metadata` to `create-table` message\n// - Adds `tableMetadata` to `add-column` message\n// - Adds `table-update-metadata` message\n// v6: v0.26\n// - Adds support for `backfill` messages\n// v6: v1.0.1 (backwards compatible, no version change)\n// - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n// - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n// the upstream commit timestamp for end-to-end serving lag measurement.\n// The stream is parsed in 'passthrough' mode, so an older peer ignores the\n// field, and a newer peer treats its absence (including in changes\n// replayed from the Change DB) as \"no commit time reported\".\n// v7: v1.12\n// - Merges the separate `/snapshot` and `/changes` connections into a single\n// bidirectional `/subscribe` connection (see subscribe.ts). The view-syncer\n// drives the connection with upstream `reserve-snapshot` / `start-subscription`\n// application messages; the change-streamer replies with a `reserved`\n// message and then the change stream on the same socket. This guarantees the\n// replication-manager that reserves the change log is the one that serves the\n// subscription. The legacy `/snapshot` + `/changes` endpoints remain for\n// v4-v6 subscribers during rollout.\n// - Drops the legacy \"initial\" and \"logsChangeStream\" query parameters from the\n// subscriber context\n// v8:\n// - Adds `backfills` to the `start-subscription` context, with which the\n// subscriber reports its pending backfills (and their progress).\n// - Adds `progressMarks` to `backfill` and `backfill-completed` messages,\n// which subscribers use to ignore backfill data that would otherwise\n// create a gap (see `acceptBackfill()`).\n// Both fields are optional in the schemas so that v7 peers (and changes\n// persisted in the change-log by v7 replication-managers) remain compatible,\n// but they should be considered non-optional as of v8.\n\nexport const PROTOCOL_VERSION = 8;\n\nexport type SubscriberContext = {\n /\n The supported change-streamer protocol version.\n /\n protocolVersion: number;\n\n /\n Task ID. This is used to link the request with a preceding snapshot\n * reservation.\n /\n taskID: string;\n\n /\n Subscriber id. This is only used for debugging.\n /\n id: string;\n\n /\n The ReplicatorMode of the subscriber. 'backup' indicates that the\n * subscriber is local to the `change-streamer` in the `replication-manager`,\n * while 'serving' indicates that user-facing requests depend on the subscriber.\n /\n mode: ReplicatorMode;\n\n /\n The ChangeStreamer will return an Error if the subscriber is\n * on a different replica version (i.e. the initial snapshot associated\n * with the replication slot).\n /\n replicaVersion: string;\n\n /\n The watermark up to which the subscriber is up to date.\n * Only changes after the watermark will be streamed.\n /\n watermark: string;\n\n /\n Whether the subscriber supports batched WebSocket frames.\n /\n wsBatched?: boolean \| undefined;\n\n /\n The subscriber's pending backfills as of its `watermark`. Reported by\n * subscribers as of protocol v8. `undefined` for older subscribers.\n /\n backfills?: BackfillRequest[] \| undefined;\n\n /\n Whether the subscriber understands the `draining` status, i.e. it will\n * switch over to another replication-manager when the current one shuts\n * down. A draining replication-manager waits for drainable subscribers to\n * leave before exiting.\n /\n drainable?: boolean \| undefined;\n\n /\n Whether the subscriber is switching over from another subscription with\n * a replica that it is currently serving. If the replica version is\n * incompatible, such a subscriber is held (see {@link Status.restore})\n * rather than being immediately rejected. Subscribers that are booting up\n * never set this.\n /\n switchover?: boolean \| undefined;\n};\n\n/\n The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n /\nexport const statusSchema = v.object({\n tag: v.literal('status'),\n\n lagReport: v\n .object({\n lastTimings: changeSourceTimingsSchema.optional(),\n nextSendTimeMs: v.number(),\n })\n .optional(),\n\n /\n Sent by a replication-manager that is shutting down to its (drainable)\n * serving subscribers. The subscriber should switch over to another\n * replication-manager (see {@link SubscriberContext.switchover}) while\n * continuing to consume this subscription until the switchover succeeds.\n /\n draining: v.boolean().optional(),\n\n /\n Sent on a {@link SubscriberContext.switchover switchover} subscription\n * to a replication-manager running an incompatible replica version.\n * Rather than being immediately rejected with\n * {@link ErrorType.WrongReplicaVersion}, the subscriber is held until it\n * is granted permission to restore the replication-manager's backup,\n * which limits the number of subscribers that are restoring (and thus not\n * serving) at the same time.\n \n * `queued`: the subscriber is waiting for permission to restore. It\n * should continue consuming its existing subscription (if any).\n * * `granted`: the subscriber should (gracefully) restart to restore a\n * new replica.\n /\n restore: v\n .union(\n v.object({state: v.literal('queued'), position: v.number()}),\n v.object({state: v.literal('granted')}),\n )\n .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/\n A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n /\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n type: v.number(), // ErrorType\n message: v.string().optional(),\n /\n The replica version of the replication-manager, sent with\n * {@link ErrorType.WrongReplicaVersion} so that a subscriber with a newer\n * replica can avoid restoring an older one.\n /\n replicaVersion: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n statusMessageSchema,\n changeStreamDataSchema,\n errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n switch (val) {\n case ErrorType.WrongReplicaVersion:\n return 'WrongReplicaVersion';\n case ErrorType.WatermarkTooOld:\n return 'WatermarkTooOld';\n case ErrorType.Unknown:\n return 'Unknown';\n default:\n return 'Unknown';\n }\n}\n\n/\n A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n \n A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n /\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\n/* A downstream message and its approximate serialized transport size. /\nexport type SizedDownstream = Sized<Downstream>;\n\nexport interface ChangeStreamerService\n extends Omit<ChangeStreamer, 'subscribe'>, Service {\n /\n The server-side interface overrides `subscribe()` to return a stream\n * of already-stringified {@link Downstream} payloads or pre-serialized Buffers.\n \n If a `downstream` subscription is supplied it is used directly as the\n * returned Source. Otherwise, a new one is created.\n /\n subscribe(\n ctx: SubscriberContext,\n downstream?: Subscription<string \| PreSerializedBatch>,\n ): Promise<Source<string \| PreSerialized>>;\n\n /\n Starts a snapshot reservation to preserve change-log entries while\n * a soon-to-be-subscriber downloads the current replica backup. Once\n * the change-streamer knows that a backup is available\n * (via {@link trackBackupWatermark}), it sends a confirmation\n * SnapshotMessage and reserves the corresponding entries in the change-log\n * until the client with that `taskID` subscribes, or if the snapshot\n * connection terminates.\n /\n startSnapshotReservation(\n taskID: string,\n opts?: {localReplica?: LocalReplica \| undefined},\n ): Promise<Source<SnapshotMessage>>;\n\n /\n Signals (drainable) serving subscribers to switch over to another\n * replication-manager, and resolves when they have all disconnected.\n * Subscribers continue to be served in the meantime.\n /\n drain?(): Promise<void>;\n\n /\n Informs the change-streamer of the watermark up to which the replica has\n * been backed up. This serves two purposes:\n * - It serves as the maximum watermark up to which the change-log can be\n * purged. Note that the change-streamer also takes into account the\n * positions of its subscribers and snapshot reservations when purging\n * changes in the change-log.\n * - In RMv2, this ACK's the upstream change-source to allow its buffer of\n * changes (e.g. the replication_slot) to advance.\n */\n trackBackupWatermark(watermark: string): void;\n}\n"],"mappings":";;;;;;;;AA2MA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;;;;;;;CAQZ,UAAU,eAAE,QAAQ,EAAE,SAAS;;;;;;;;;;;;;;;CAgB/B,SAAS,eACN,MACC,eAAE,OAAO;EAAC,OAAO,eAAE,QAAQ,QAAQ;EAAG,UAAU,eAAE,OAAO;CAAC,CAAC,GAC3D,eAAE,OAAO,EAAC,OAAO,eAAE,QAAQ,SAAS,EAAC,CAAC,CACxC,EACC,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;;;;;;CAM7B,gBAAgB,eAAE,OAAO,EAAE,SAAS;AACtC,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}
1	+ {"version":3,"file":"change-streamer.js","names":["v"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {PreSerialized, Sized, Source} from '../../types/streams.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n changeStreamDataSchema,\n type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {LocalReplica, SubscribeDownstream} from './subscribe.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/*\n Internally all downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering while replaying stored changes and\n * filtering changes already seen by a subscriber.\n \n Only commit watermarks are exposed to subscribers. The JSON string is the\n * canonical serialization shared by storage and live forwarding.\n /\nexport type WatermarkedChange = [\n watermark: string,\n tag: ChangeTag,\n json: string,\n];\n\n/\n The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n \n * Changes from the upstream source are immediately forwarded to\n * connected subscribers to minimize latency.\n \n * They are then stored in a separate DB to facilitate catchup\n * of connecting subscribers that are behind.\n \n * Acknowledgements are sent upstream after they are successfully\n * stored.\n \n Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n \n However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n \n The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n /\nexport interface ChangeStreamer {\n /\n Subscribes to changes based on the supplied subscriber `ctx`,\n * which indicates the watermark at which the subscriber is up to\n * date. Each result contains parsed, validated data and its approximate\n * serialized transport size.\n /\n subscribe(\n ctx: SubscriberContext,\n ): Promise<Source<Sized<SubscribeDownstream>>>;\n}\n\n// v1: v0.18\n// - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n// - Adds the \"status\" message which is initially used to signal that the\n// subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n// - Adds the \"taskID\" to the subscription context, and support for\n// the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n// - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n// status request so that a subscriber can verify whether its replica,\n// whether it be restored or existing in a permanent volume, is compatible\n// with the change-streamer.\n// v5: v0.26\n// - Moves relation.keyColumns and relation.replicaIdentity to\n// relation.rowKey: { columns, type }.\n// - Adds `metadata` to `create-table` message\n// - Adds `tableMetadata` to `add-column` message\n// - Adds `table-update-metadata` message\n// v6: v0.26\n// - Adds support for `backfill` messages\n// v6: v1.0.1 (backwards compatible, no version change)\n// - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n// - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n// the upstream commit timestamp for end-to-end serving lag measurement.\n// The stream is parsed in 'passthrough' mode, so an older peer ignores the\n// field, and a newer peer treats its absence (including in changes\n// replayed from the Change DB) as \"no commit time reported\".\n// v7: v1.12\n// - Merges the separate `/snapshot` and `/changes` connections into a single\n// bidirectional `/subscribe` connection (see subscribe.ts). The view-syncer\n// drives the connection with upstream `reserve-snapshot` / `start-subscription`\n// application messages; the change-streamer replies with a `reserved`\n// message and then the change stream on the same socket. This guarantees the\n// replication-manager that reserves the change log is the one that serves the\n// subscription. The legacy `/snapshot` + `/changes` endpoints remain for\n// v4-v6 subscribers during rollout.\n// - Drops the legacy \"initial\" and \"logsChangeStream\" query parameters from the\n// subscriber context\n// v8:\n// - Adds `backfills` to the `start-subscription` context, with which the\n// subscriber reports its pending backfills (and their progress).\n// - Adds `progressMarks` to `backfill` and `backfill-completed` messages,\n// which subscribers use to ignore backfill data that would otherwise\n// create a gap (see `acceptBackfill()`).\n// Both fields are optional in the schemas so that v7 peers (and changes\n// persisted in the change-log by v7 replication-managers) remain compatible,\n// but they should be considered non-optional as of v8.\n\nexport const PROTOCOL_VERSION = 8;\n\nexport type SubscriberContext = {\n /\n The supported change-streamer protocol version.\n /\n protocolVersion: number;\n\n /\n Task ID. This is used to link the request with a preceding snapshot\n * reservation.\n /\n taskID: string;\n\n /\n Subscriber id. This is only used for debugging.\n /\n id: string;\n\n /\n The ReplicatorMode of the subscriber. 'backup' indicates that the\n * subscriber is local to the `change-streamer` in the `replication-manager`,\n * while 'serving' indicates that user-facing requests depend on the subscriber.\n /\n mode: ReplicatorMode;\n\n /\n The ChangeStreamer will return an Error if the subscriber is\n * on a different replica version (i.e. the initial snapshot associated\n * with the replication slot).\n /\n replicaVersion: string;\n\n /\n The watermark up to which the subscriber is up to date.\n * Only changes after the watermark will be streamed.\n /\n watermark: string;\n\n /\n Whether the subscriber supports batched WebSocket frames.\n /\n wsBatched?: boolean \| undefined;\n\n /\n The subscriber's pending backfills as of its `watermark`. Reported by\n * subscribers as of protocol v8. `undefined` for older subscribers.\n /\n backfills?: BackfillRequest[] \| undefined;\n\n /\n Whether the subscriber understands the `draining` status, i.e. it will\n * switch over to another replication-manager when the current one shuts\n * down. A draining replication-manager waits for drainable subscribers to\n * leave before exiting.\n /\n drainable?: boolean \| undefined;\n\n /\n Whether the subscriber is switching over from another subscription with\n * a replica that it is currently serving. If the replica version is\n * incompatible, such a subscriber is held (see {@link Status.restore})\n * rather than being immediately rejected. Subscribers that are booting up\n * never set this.\n /\n switchover?: boolean \| undefined;\n};\n\n/\n The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n /\nexport const statusSchema = v.object({\n tag: v.literal('status'),\n\n lagReport: v\n .object({\n lastTimings: changeSourceTimingsSchema.optional(),\n nextSendTimeMs: v.number(),\n })\n .optional(),\n\n /\n Sent by a replication-manager that is shutting down to its (drainable)\n * serving subscribers. The subscriber should switch over to another\n * replication-manager (see {@link SubscriberContext.switchover}) while\n * continuing to consume this subscription until the switchover succeeds.\n /\n draining: v.boolean().optional(),\n\n /\n Sent on a {@link SubscriberContext.switchover switchover} subscription\n * to a replication-manager running an incompatible replica version.\n * Rather than being immediately rejected with\n * {@link ErrorType.WrongReplicaVersion}, the subscriber is held until it\n * is granted permission to restore the replication-manager's backup,\n * which limits the number of subscribers that are restoring (and thus not\n * serving) at the same time.\n \n * `queued`: the subscriber is waiting for permission to restore. It\n * should continue consuming its existing subscription (if any).\n * * `granted`: the subscriber should (gracefully) restart to restore a\n * new replica.\n /\n restore: v\n .union(\n v.object({state: v.literal('queued'), position: v.number()}),\n v.object({state: v.literal('granted')}),\n )\n .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/\n A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n /\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n type: v.number(), // ErrorType\n message: v.string().optional(),\n /\n The replica version of the replication-manager, sent with\n * {@link ErrorType.WrongReplicaVersion} so that a subscriber with a newer\n * replica can avoid restoring an older one.\n /\n replicaVersion: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n statusMessageSchema,\n changeStreamDataSchema,\n errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n switch (val) {\n case ErrorType.WrongReplicaVersion:\n return 'WrongReplicaVersion';\n case ErrorType.WatermarkTooOld:\n return 'WatermarkTooOld';\n case ErrorType.Unknown:\n return 'Unknown';\n default:\n return 'Unknown';\n }\n}\n\n/\n A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n \n A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n /\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\n/* A downstream message and its approximate serialized transport size. /\nexport type SizedDownstream = Sized<Downstream>;\n\nexport interface ChangeStreamerService\n extends Omit<ChangeStreamer, 'subscribe'>, Service {\n /\n The server-side interface overrides `subscribe()` to return a stream\n * of already-stringified {@link Downstream} payloads or pre-serialized Buffers.\n \n If a `downstream` subscription is supplied it is used directly as the\n * returned Source. Otherwise, a new one is created.\n /\n subscribe(\n ctx: SubscriberContext,\n downstream?: Subscription<string \| PreSerializedBatch>,\n ): Promise<Source<string \| PreSerialized>>;\n\n /\n Starts a snapshot reservation to preserve change-log entries while\n * a soon-to-be-subscriber downloads the current replica backup. Once\n * the change-streamer knows that a backup is available\n * (via {@link trackBackupWatermark}), it sends a confirmation\n * SnapshotMessage and reserves the corresponding entries in the change-log\n * until the client with that `taskID` subscribes, or if the snapshot\n * connection terminates.\n /\n startSnapshotReservation(\n taskID: string,\n opts?: {localReplica?: LocalReplica \| undefined},\n ): Promise<Source<SnapshotMessage>>;\n\n /\n Signals (drainable) serving subscribers to switch over to another\n * replication-manager, and resolves when they have all disconnected.\n * Subscribers continue to be served in the meantime.\n /\n drain?(): Promise<void>;\n\n /\n Informs the change-streamer of the watermark up to which the replica has\n * been backed up. This serves two purposes:\n * - It serves as the maximum watermark up to which the change-log can be\n * purged. Note that the change-streamer also takes into account the\n * positions of its subscribers and snapshot reservations when purging\n * changes in the change-log.\n * - In RMv2, this ACK's the upstream change-source to allow its buffer of\n * changes (e.g. the replication_slot) to advance.\n */\n trackBackupWatermark(watermark: string): void;\n}\n"],"mappings":";;;;;;;;AA2MA,IAAa,eAAeA,eAAE,OAAO;CACnC,KAAKA,eAAE,QAAQ,QAAQ;CAEvB,WAAWA,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgBA,eAAE,OAAO;CAC3B,CAAC,CAAC,CACD,SAAS;;;;;;;CAQZ,UAAUA,eAAE,QAAQ,CAAC,CAAC,SAAS;;;;;;;;;;;;;;;CAgB/B,SAASA,eACN,MACCA,eAAE,OAAO;EAAC,OAAOA,eAAE,QAAQ,QAAQ;EAAG,UAAUA,eAAE,OAAO;CAAC,CAAC,GAC3DA,eAAE,OAAO,EAAC,OAAOA,eAAE,QAAQ,SAAS,EAAC,CAAC,CACxC,CAAC,CACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsBA,eAAE,MAAM,CAACA,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0BA,eAAE,OAAO;CACvC,MAAMA,eAAE,OAAO;CACf,SAASA,eAAE,OAAO,CAAC,CAAC,SAAS;;;;;;CAM7B,gBAAgBA,eAAE,OAAO,CAAC,CAAC,SAAS;AACtC,CAAC;AAID,IAAM,cAAcA,eAAE,MAAM,CAACA,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmBA,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}

package/out/zero-cache/src/services/change-streamer/drain-tracker.d.ts CHANGED Viewed

@@ -7,10 +7,10 @@ import type { Subscriber } from './subscriber.ts';
  * change-streamer, i.e. those that switch over to another
  * replication-manager when this one drains.
  *
- * Upon {@link drain}, drainable subscribers are sent a `draining` status
- * (as are drainable subscribers that subscribe while draining), and the
- * returned promise resolves once they have all disconnected. Subscribers
- * continue to be served in the meantime.
+ * Upon {@link drain}, drainable subscribers are sent a `draining` status,
+ * and the returned promise resolves once they have all disconnected.
+ * Subscribers continue to be served in the meantime. (New subscribers are
+ * not accepted while draining.)
  */
 export declare class DrainTracker {
     #private;

package/out/zero-cache/src/services/change-streamer/drain-tracker.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"drain-tracker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/drain-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAIhD;;;;;;;;;GASG;AACH,qBAAa,YAAY;;IAOvB,YACE,EAAE,EAAE,UAAU,EAEd,aAAa,GAAE,OAAO,WAAyB,EAC/C,eAAe,GAAE,OAAO,aAA6B,EAKtD;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,QAAQ,CACN,UAAU,EAAE,UAAU,EACtB,UAAU,EAAE,YAAY,CAAC,MAAM,GAAG,kBAAkB,CAAC,~~QActD~~;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAuBrB;IAED,uEAAuE;IACvE,IAAI,SAEH;CAQF"}
1	+ {"version":3,"file":"drain-tracker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/drain-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAIhD;;;;;;;;;GASG;AACH,qBAAa,YAAY;;IAOvB,YACE,EAAE,EAAE,UAAU,EAEd,aAAa,GAAE,OAAO,WAAyB,EAC/C,eAAe,GAAE,OAAO,aAA6B,EAKtD;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,QAAQ,CACN,UAAU,EAAE,UAAU,EACtB,UAAU,EAAE,YAAY,CAAC,MAAM,GAAG,kBAAkB,CAAC,QAWtD;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAuBrB;IAED,uEAAuE;IACvE,IAAI,SAEH;CAQF"}

package/out/zero-cache/src/services/change-streamer/drain-tracker.js CHANGED Viewed

@@ -6,10 +6,10 @@ var PROGRESS_LOG_INTERVAL_MS = 3e4;
 * change-streamer, i.e. those that switch over to another
 * replication-manager when this one drains.
 *
-* Upon {@link drain}, drainable subscribers are sent a `draining` status
-* (as are drainable subscribers that subscribe while draining), and the
-* returned promise resolves once they have all disconnected. Subscribers
-* continue to be served in the meantime.
+* Upon {@link drain}, drainable subscribers are sent a `draining` status,
+* and the returned promise resolves once they have all disconnected.
+* Subscribers continue to be served in the meantime. (New subscribers are
+* not accepted while draining.)
 */
 var DrainTracker = class {
 	#lc;
@@ -28,10 +28,6 @@ var DrainTracker = class {
 	register(subscriber, downstream) {
 		if (!subscriber.drainable) return;
 		this.#subscribers.add(subscriber);
-		if (this.#draining) subscriber.sendControlStatus({
-			tag: "status",
-			draining: true
-		});
 		downstream.addCloseHandler(() => {
 			if (this.#subscribers.delete(subscriber)) this.#checkDrained();
 		});

package/out/zero-cache/src/services/change-streamer/drain-tracker.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"drain-tracker.js","names":[~~"#lc","#subscribers","#setInterval","#clearInterval","#draining","#checkDrained"~~],"sources":["../../../../../../zero-cache/src/services/change-streamer/drain-tracker.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nconst PROGRESS_LOG_INTERVAL_MS = 30_000;\n\n/*\n Tracks the {@link Subscriber.drainable drainable} subscribers of the\n * change-streamer, i.e. those that switch over to another\n * replication-manager when this one drains.\n \n Upon {@link drain}, drainable subscribers are sent a `draining` status\n * ~~(as are drainable subscribers that subscribe while draining),~~ and the\n * returned promise resolves once they have all disconnected~~. Subscribers\~~n * continue to be served in the meantime.\n /\nexport class DrainTracker {\n readonly #lc: LogContext;\n readonly #subscribers = new Set<Subscriber>();\n readonly #setInterval: typeof setInterval;\n readonly #clearInterval: typeof clearInterval;\n #draining: Resolver<void> \| undefined;\n\n constructor(\n lc: LogContext,\n // Injectable timers, for deterministic testing.\n setIntervalFn: typeof setInterval = setInterval,\n clearIntervalFn: typeof clearInterval = clearInterval,\n ) {\n this.#lc = lc.withContext('component', 'drain-tracker');\n this.#setInterval = setIntervalFn;\n this.#clearInterval = clearIntervalFn;\n }\n\n get draining(): boolean {\n return this.#draining !== undefined;\n }\n\n register(\n subscriber: Subscriber,\n downstream: Subscription<string \| PreSerializedBatch>,\n ) {\n if (!subscriber.drainable) {\n return;\n }\n this.#subscribers.add(subscriber);\n ~~if (this.#draining) {\n subscriber.sendControlStatus({tag: 'status', draining: true});\n }\n~~ downstream.addCloseHandler(() => {\n if (this.#subscribers.delete(subscriber)) {\n this.#checkDrained();\n }\n });\n }\n\n drain(): Promise<void> {\n if (!this.#draining) {\n const draining = (this.#draining = resolver<void>());\n this.#lc.info?.(\n `draining: signaling ${this.#subscribers.size} subscribers ` +\n `to switch over`,\n );\n for (const sub of this.#subscribers) {\n sub.sendControlStatus({tag: 'status', draining: true});\n }\n const progress = this.#setInterval(\n () =>\n this.#lc.info?.(\n `draining: waiting for ${this.#subscribers.size} ` +\n `subscribers to switch over`,\n Array.from(this.#subscribers, sub => sub.id),\n ),\n PROGRESS_LOG_INTERVAL_MS,\n );\n void draining.promise.finally(() => this.#clearInterval(progress));\n this.#checkDrained();\n }\n return this.#draining.promise;\n }\n\n /* Resolves any pending drain, e.g. when the change-streamer stops. */\n stop() {\n this.#draining?.resolve();\n }\n\n #checkDrained() {\n if (this.#draining && this.#subscribers.size === 0) {\n this.#lc.info?.(`drained all subscribers`);\n this.#draining.resolve();\n }\n }\n}\n"],"mappings":";;AAMA,IAAM,2BAA2B;;;;;;;;;;;AAYjC,IAAa,eAAb,MAA0B;CACxB;CACA,+BAAwB,IAAI,IAAgB;CAC5C;CACA;CACA;CAEA,YACE,IAEA,gBAAoC,aACpC,kBAAwC,eACxC;EACA,~~KAAKA~~,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,~~KAAKE~~,eAAe;EACpB,~~KAAKC~~,iBAAiB;CACxB;CAEA,IAAI,WAAoB;EACtB,OAAO,~~KAAKC~~,cAAc,KAAA;CAC5B;CAEA,SACE,YACA,YACA;EACA,IAAI,CAAC,WAAW,WACd;EAEF,~~KAAKH~~,aAAa,IAAI,UAAU;EAChC,~~IAAI,KAAKG,WACP,~~WAAW,~~kBAAkB;GAAC,KAAK;GAAU,UAAU;EAAI,CAAC;EAE9D,WAAW,~~sBAAsB;GAC/B,IAAI,~~KAAKH~~,aAAa,OAAO,UAAU,GACrC,~~KAAKI~~,cAAc;EAEvB,CAAC;CACH;CAEA,QAAuB;EACrB,IAAI,CAAC,~~KAAKD~~,WAAW;GACnB,MAAM,WAAY,~~KAAKA~~,YAAY,SAAe;GAClD,~~KAAKJ~~,IAAI,OACP,uBAAuB,~~KAAKC~~,aAAa,KAAK,4BAEhD;GACA,KAAK,MAAM,OAAO,~~KAAKA~~,cACrB,IAAI,kBAAkB;IAAC,KAAK;IAAU,UAAU;GAAI,CAAC;GAEvD,MAAM,WAAW,~~KAAKC~~,mBAElB,~~KAAKF~~,IAAI,OACP,yBAAyB,~~KAAKC~~,aAAa,KAAK,8BAEhD,MAAM,KAAK,~~KAAKA~~,eAAc,QAAO,IAAI,EAAE,CAC7C,GACF,wBACF;GACA,SAAc,QAAQ,cAAc,~~KAAKE~~,eAAe,QAAQ,CAAC;GACjE,~~KAAKE~~,cAAc;EACrB;EACA,OAAO,~~KAAKD~~,UAAU;CACxB;;CAGA,OAAO;EACL,~~KAAKA~~,WAAW,QAAQ;CAC1B;CAEA,gBAAgB;EACd,IAAI,~~KAAKA~~,aAAa,~~KAAKH~~,aAAa,SAAS,GAAG;GAClD,~~KAAKD~~,IAAI,OAAO,yBAAyB;GACzC,~~KAAKI~~,UAAU,QAAQ;EACzB;CACF;AACF"}
1	+ {"version":3,"file":"drain-tracker.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/drain-tracker.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nconst PROGRESS_LOG_INTERVAL_MS = 30_000;\n\n/*\n Tracks the {@link Subscriber.drainable drainable} subscribers of the\n * change-streamer, i.e. those that switch over to another\n * replication-manager when this one drains.\n \n Upon {@link drain}, drainable subscribers are sent a `draining` status,\n * and the returned promise resolves once they have all disconnected.\n * Subscribers continue to be served in the meantime. (New subscribers are\n * not accepted while draining.)\n /\nexport class DrainTracker {\n readonly #lc: LogContext;\n readonly #subscribers = new Set<Subscriber>();\n readonly #setInterval: typeof setInterval;\n readonly #clearInterval: typeof clearInterval;\n #draining: Resolver<void> \| undefined;\n\n constructor(\n lc: LogContext,\n // Injectable timers, for deterministic testing.\n setIntervalFn: typeof setInterval = setInterval,\n clearIntervalFn: typeof clearInterval = clearInterval,\n ) {\n this.#lc = lc.withContext('component', 'drain-tracker');\n this.#setInterval = setIntervalFn;\n this.#clearInterval = clearIntervalFn;\n }\n\n get draining(): boolean {\n return this.#draining !== undefined;\n }\n\n register(\n subscriber: Subscriber,\n downstream: Subscription<string \| PreSerializedBatch>,\n ) {\n if (!subscriber.drainable) {\n return;\n }\n this.#subscribers.add(subscriber);\n downstream.addCloseHandler(() => {\n if (this.#subscribers.delete(subscriber)) {\n this.#checkDrained();\n }\n });\n }\n\n drain(): Promise<void> {\n if (!this.#draining) {\n const draining = (this.#draining = resolver<void>());\n this.#lc.info?.(\n `draining: signaling ${this.#subscribers.size} subscribers ` +\n `to switch over`,\n );\n for (const sub of this.#subscribers) {\n sub.sendControlStatus({tag: 'status', draining: true});\n }\n const progress = this.#setInterval(\n () =>\n this.#lc.info?.(\n `draining: waiting for ${this.#subscribers.size} ` +\n `subscribers to switch over`,\n Array.from(this.#subscribers, sub => sub.id),\n ),\n PROGRESS_LOG_INTERVAL_MS,\n );\n void draining.promise.finally(() => this.#clearInterval(progress));\n this.#checkDrained();\n }\n return this.#draining.promise;\n }\n\n /* Resolves any pending drain, e.g. when the change-streamer stops. */\n stop() {\n this.#draining?.resolve();\n }\n\n #checkDrained() {\n if (this.#draining && this.#subscribers.size === 0) {\n this.#lc.info?.(`drained all subscribers`);\n this.#draining.resolve();\n }\n }\n}\n"],"mappings":";;AAMA,IAAM,2BAA2B;;;;;;;;;;;AAYjC,IAAa,eAAb,MAA0B;CACxB;CACA,+BAAwB,IAAI,IAAgB;CAC5C;CACA;CACA;CAEA,YACE,IAEA,gBAAoC,aACpC,kBAAwC,eACxC;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAK,eAAe;EACpB,KAAK,iBAAiB;CACxB;CAEA,IAAI,WAAoB;EACtB,OAAO,KAAK,cAAc,KAAA;CAC5B;CAEA,SACE,YACA,YACA;EACA,IAAI,CAAC,WAAW,WACd;EAEF,KAAK,aAAa,IAAI,UAAU;EAChC,WAAW,sBAAsB;GAC/B,IAAI,KAAK,aAAa,OAAO,UAAU,GACrC,KAAK,cAAc;EAEvB,CAAC;CACH;CAEA,QAAuB;EACrB,IAAI,CAAC,KAAK,WAAW;GACnB,MAAM,WAAY,KAAK,YAAY,SAAe;GAClD,KAAK,IAAI,OACP,uBAAuB,KAAK,aAAa,KAAK,4BAEhD;GACA,KAAK,MAAM,OAAO,KAAK,cACrB,IAAI,kBAAkB;IAAC,KAAK;IAAU,UAAU;GAAI,CAAC;GAEvD,MAAM,WAAW,KAAK,mBAElB,KAAK,IAAI,OACP,yBAAyB,KAAK,aAAa,KAAK,8BAEhD,MAAM,KAAK,KAAK,eAAc,QAAO,IAAI,EAAE,CAC7C,GACF,wBACF;GACA,SAAc,QAAQ,cAAc,KAAK,eAAe,QAAQ,CAAC;GACjE,KAAK,cAAc;EACrB;EACA,OAAO,KAAK,UAAU;CACxB;;CAGA,OAAO;EACL,KAAK,WAAW,QAAQ;CAC1B;CAEA,gBAAgB;EACd,IAAI,KAAK,aAAa,KAAK,aAAa,SAAS,GAAG;GAClD,KAAK,IAAI,OAAO,yBAAyB;GACzC,KAAK,UAAU,QAAQ;EACzB;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/forwarder.js CHANGED Viewed

@@ -68,16 +68,18 @@ var Forwarder = class {
 		if (this.#flowControlSlowSubscriberGracePeriodMs && timedOut?.size && slowestOnTime) for (const [sub, stats] of timedOut.entries()) {
 			const rate = stats.processRate < slowestOnTime.processRate ? "lagging" : "catching-up";
 			const laggingDuration = sub.reportChangeRate(now, rate);
-			if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) if (sub.mode === "backup") {
-				this.#backupLagEventCount++;
-				const grace = this.#flowControlSlowSubscriberGracePeriodMs;
-				if (this.#lastBackupLagWarnMs === void 0 || now - this.#lastBackupLagWarnMs >= grace) {
-					this.#lastBackupLagWarnMs = now;
-					this.#lc.warn?.(`backup subscriber ${sub.id} has lagged for ${laggingDuration}ms (${this.#backupLagEventCount} lag events so far). this is only expected during initial catchup`, { stats });
+			if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {
+				if (sub.mode === "backup") {
+					this.#backupLagEventCount++;
+					const grace = this.#flowControlSlowSubscriberGracePeriodMs;
+					if (this.#lastBackupLagWarnMs === void 0 || now - this.#lastBackupLagWarnMs >= grace) {
+						this.#lastBackupLagWarnMs = now;
+						this.#lc.warn?.(`backup subscriber ${sub.id} has lagged for ${laggingDuration}ms (${this.#backupLagEventCount} lag events so far). this is only expected during initial catchup`, { stats });
+					}
+				} else {
+					this.#lc.warn?.(`serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`, { stats });
+					sub.close(3, `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`);
 				}
-			} else {
-				this.#lc.warn?.(`serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`, { stats });
-				sub.close(3, `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`);
 			}
 		}
 	}
@@ -188,7 +190,6 @@ var Forwarder = class {
 				this.#inTransaction = false;
 				for (const sub of this.#queued.values()) this.#active.add(sub);
 				this.#queued.clear();
-				break;
 		}
 	}
 	getAcks() {

package/out/zero-cache/src/services/change-streamer/forwarder.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#backupLagEventCount","#lastBackupLagWarnMs","#flushPendingWithoutTracking","#inTransaction","#pending","#updateActiveSubscribers","#flushScheduled","#flushImmediateId","#drainPending"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n getOrCreateCounter,\n getOrCreateGauge,\n getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n flowControlConsensusTimeoutProportion: number;\n flowControlSlowSubscriberGracePeriodMs?: number \| undefined;\n // Injectable timers, for deterministic testing.\n setTimeoutFn?: typeof setTimeout \| undefined;\n clearTimeoutFn?: typeof clearTimeout \| undefined;\n};\n\nconst FORWARD_BATCH_SIZE = 64;\n\nexport class Forwarder {\n readonly #lc: LogContext;\n readonly #setTimeout: typeof setTimeout;\n readonly #clearTimeout: typeof clearTimeout;\n readonly #active = new Set<Subscriber>();\n readonly #queued = new Set<Subscriber>();\n readonly #earlyReleaseOptions: EarlyReleaseOptions \| undefined;\n readonly #flowControlSlowSubscriberGracePeriodMs: number \| undefined;\n\n readonly #flowControlWaits = getOrCreateCounter(\n 'replication',\n 'flow_control.waits',\n 'Flow-control checkpoints completed, labeled by release mode.',\n );\n readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n 'replication',\n 'flow_control.wait_duration',\n 'Time replication waits at flow-control checkpoints.',\n );\n #inTransaction = false;\n\n #pending: WatermarkedChange[] = [];\n #flushScheduled = false;\n #flushImmediateId: NodeJS.Immediate \| undefined = undefined;\n\n #currentBroadcast: Broadcast \| undefined;\n #progressMonitor: NodeJS.Timeout \| undefined;\n\n // The backup-replicator is never disconnected as a laggard, so its lag\n // detections are logged (rather than acted upon). Track the running total and\n // throttle the warning to at most once per grace period to avoid spamming the\n // log for the duration of a slow catchup.\n #backupLagEventCount = 0;\n #lastBackupLagWarnMs: number \| undefined;\n\n constructor(\n lc: LogContext,\n opts: ProgressMonitorOptions = {\n flowControlConsensusTimeoutProportion: 2.0,\n },\n ) {\n this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n this.#earlyReleaseOptions =\n opts.flowControlConsensusTimeoutProportion < 0\n ? undefined\n : {\n consensusTimeoutProportion:\n opts.flowControlConsensusTimeoutProportion,\n setTimeoutFn: this.#setTimeout,\n clearTimeoutFn: this.#clearTimeout,\n };\n this.#flowControlSlowSubscriberGracePeriodMs =\n opts.flowControlSlowSubscriberGracePeriodMs;\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.active_subscribers',\n 'Active change-stream subscribers receiving live changes.',\n ).addCallback(result => result.observe(this.#active.size));\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.queued_subscribers',\n 'Change-stream subscribers waiting for the current transaction to finish before activation.',\n ).addCallback(result => result.observe(this.#queued.size));\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.pending_messages',\n 'Downstream change-stream messages not yet acked by subscribers.',\n ).addCallback(result =>\n result.observe(this.#subscriberStats().pendingMessages),\n );\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.backlog_messages',\n 'Live change-stream messages buffered while subscribers catch up.',\n ).addCallback(result =>\n result.observe(this.#subscriberStats().backlogMessages),\n );\n\n getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n description:\n 'Live change-stream bytes buffered while subscribers catch up.',\n unit: 'By',\n }).addCallback(result =>\n result.observe(this.#subscriberStats().backlogBytes),\n );\n\n getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n description:\n 'Maximum live change-stream bytes buffered by a single subscriber.',\n unit: 'By',\n }).addCallback(result =>\n result.observe(this.#subscriberStats().maxBacklogBytes),\n );\n }\n\n startProgressMonitor() {\n clearInterval(this.#progressMonitor);\n this.#progressMonitor = setInterval(\n () => this.checkSubscriberProgress(performance.now()),\n 1000,\n );\n }\n\n /*\n Runs one progress-monitor tick: samples each active subscriber's change\n * rate, logs slow broadcasts, and disconnects subscribers that have been\n * continuously lagging for longer than the configured grace period.\n \n Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n * than private) so tests can drive a single tick with an explicit `now`.\n /\n checkSubscriberProgress(now: number) {\n let timedOut: Map<Subscriber, SubscriberStats> \| undefined;\n let slowestOnTime: SubscriberStats \| undefined;\n\n for (const sub of this.#active) {\n const stats = sub.sampleProcessRate(now).getStats();\n if (stats.missedLastTimeout) {\n (timedOut ??= new Map()).set(sub, stats);\n } else if (\n stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n ) {\n slowestOnTime = stats;\n }\n }\n this.#currentBroadcast?.logSlowRequests(now);\n\n if (\n this.#flowControlSlowSubscriberGracePeriodMs &&\n timedOut?.size &&\n slowestOnTime\n ) {\n for (const [sub, stats] of timedOut.entries()) {\n const rate =\n stats.processRate < slowestOnTime.processRate\n ? 'lagging'\n : 'catching-up';\n const laggingDuration = sub.reportChangeRate(now, rate);\n if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n if (sub.mode === 'backup') {\n // The backup-replicator can only be detected as a laggard during\n // initial catchup; after that it is required for consensus. If it\n // triggers laggard detection in the former case, just log and\n // continue; as disconnecting it would otherwise kill the\n // change-streamer task altogether.\n this.#backupLagEventCount++;\n const grace = this.#flowControlSlowSubscriberGracePeriodMs;\n if (\n this.#lastBackupLagWarnMs === undefined \|\|\n now - this.#lastBackupLagWarnMs >= grace\n ) {\n this.#lastBackupLagWarnMs = now;\n this.#lc.warn?.(\n `backup subscriber ${sub.id} has lagged for ${laggingDuration}ms ` +\n `(${this.#backupLagEventCount} lag events so far). ` +\n `this is only expected during initial catchup`,\n {stats},\n );\n }\n } else {\n this.#lc.warn?.(\n `serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n {stats},\n );\n sub.close(\n StreamTooFarBehind,\n `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n );\n }\n }\n }\n }\n }\n\n stopProgressMonitor() {\n clearInterval(this.#progressMonitor);\n this.#flushPendingWithoutTracking();\n }\n\n /\n `add()` is called in lock step with `Storer.catchup()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n /\n add(sub: Subscriber) {\n this.#flushPendingWithoutTracking();\n if (this.#inTransaction) {\n this.#queued.add(sub);\n } else {\n this.#active.add(sub);\n }\n }\n\n remove(sub: Subscriber) {\n this.#flushPendingWithoutTracking();\n this.#active.delete(sub);\n this.#queued.delete(sub);\n sub.close();\n }\n\n /\n `forward()` is called in lockstep with `Storer.store()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n \n This version of forward buffers changes across the current event loop turn\n * and dispatches them in batches. The change-streamer should call and await\n * {@link forwardWithFlowControl()} occasionally to avoid memory blowup.\n /\n forward(entry: WatermarkedChange) {\n this.#pending.push(entry);\n\n if (\n this.#queued.size > 0 &&\n (entry[1] === 'commit' \|\| entry[1] === 'rollback')\n ) {\n this.#flushPendingWithoutTracking();\n }\n this.#updateActiveSubscribers(entry[1]);\n\n if (this.#pending.length >= FORWARD_BATCH_SIZE) {\n this.#flushPendingWithoutTracking();\n } else if (!this.#flushScheduled && this.#pending.length > 0) {\n this.#flushScheduled = true;\n this.#flushImmediateId = setImmediate(() =>\n this.#flushPendingWithoutTracking(),\n );\n }\n }\n\n sendStatus(status: Status) {\n this.#flushPendingWithoutTracking();\n for (const sub of this.#active.values()) {\n sub.sendStatus(status);\n }\n }\n\n /\n The flow-control-aware equivalent of {@link forward()}, returning a\n * Promise that resolves when replication should continue.\n */\n async forwardWithFlowControl(entry: WatermarkedChange) {\n this.#pending.push(entry);\n\n const batch = this.#drainPending();\n const start = performance.now();\n const broadcast = new Broadcast(\n this.#lc,\n this.#active.values(),\n batch,\n this.#earlyReleaseOptions,\n );\n this.#updateActiveSubscribers(entry[1]);\n\n // set for progress tracking\n this.#currentBroadcast = broadcast;\n\n try {\n await broadcast.done;\n const attributes = {'release.mode': broadcast.releaseMode};\n this.#flowControlWaits.add(1, attributes);\n this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n } finally {\n // Technically #currentBroadcast may have changed, so only\n // unset if it if is still the same.\n if (this.#currentBroadcast === broadcast) {\n this.#currentBroadcast = undefined;\n }\n }\n }\n\n #drainPending(): WatermarkedChange[] {\n if (this.#flushImmediateId !== undefined) {\n clearImmediate(this.#flushImmediateId);\n this.#flushImmediateId = undefined;\n }\n this.#flushScheduled = false;\n if (this.#pending.length === 0) {\n return [];\n }\n const batch = this.#pending;\n this.#pending = [];\n return batch;\n }\n\n #flushPendingWithoutTracking() {\n const batch = this.#drainPending();\n if (batch.length > 0) {\n Broadcast.withoutTracking(this.#active.values(), batch);\n }\n }\n\n #subscriberStats() {\n let pendingMessages = 0;\n let backlogMessages = 0;\n let backlogBytes = 0;\n let maxBacklogBytes = 0;\n\n for (const sub of joinIterables(this.#active, this.#queued)) {\n const stats = sub.getStats();\n pendingMessages += stats.pending;\n backlogMessages += stats.backlog;\n backlogBytes += stats.backlogBytes;\n maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n }\n\n return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n }\n\n #updateActiveSubscribers(tag: ChangeTag) {\n switch (tag) {\n case 'begin':\n // While in a Transaction, all added subscribers are \"queued\" so that no\n // messages are forwarded to them. This state corresponds to being queued\n // for catchup in the Storer, which will retrieve historic changes\n // and call catchup() once the current transaction is committed.\n this.#inTransaction = true;\n break;\n case 'commit':\n case 'rollback':\n // Upon commit or rollback, all queued subscribers are transferred to\n // the active set. This means that they can receive messages starting\n // from the next transaction.\n //\n // Note that if catchup is still in progress (in the Storer), these messages\n // will be buffered in the backlog until catchup completes.\n this.#inTransaction = false;\n for (const sub of this.#queued.values()) {\n this.#active.add(sub);\n }\n this.#queued.clear();\n break;\n }\n }\n\n getAcks(): Set<string> {\n return new Set(\n joinIterables(\n wrapIterable(this.#active).map(s => s.acked),\n wrapIterable(this.#queued).map(s => s.acked),\n ),\n );\n }\n}\n"],"mappings":";;;;AAoBA,IAAM,qBAAqB;AAE3B,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB,WAAgC,CAAC;CACjC,kBAAkB;CAClB,oBAAkD,KAAA;CAElD;CACA;CAMA,uBAAuB;CACvB;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAC1B,IAAI,IAAI,SAAS,UAAU;IAMzB,KAAKM;IACL,MAAM,QAAQ,KAAKN;IACnB,IACE,KAAKO,yBAAyB,KAAA,KAC9B,MAAM,KAAKA,wBAAwB,OACnC;KACA,KAAKA,uBAAuB;KAC5B,KAAKb,IAAI,OACP,qBAAqB,IAAI,GAAG,kBAAkB,gBAAgB,MACxD,KAAKY,qBAAqB,oEAEhC,EAAC,MAAK,CACR;IACF;GACF,OAAO;IACL,KAAKZ,IAAI,OACP,sBAAsB,IAAI,GAAG,kBAAkB,gBAAgB,oBAC/D,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EAEJ;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;EACnC,KAAKI,6BAA6B;CACpC;;;;;;CAOA,IAAI,KAAiB;EACnB,KAAKA,6BAA6B;EAClC,IAAI,KAAKC,gBACP,KAAKX,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKW,6BAA6B;EAClC,KAAKX,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,KAAKY,SAAS,KAAK,KAAK;EAExB,IACE,KAAKZ,QAAQ,OAAO,MACnB,MAAM,OAAO,YAAY,MAAM,OAAO,aAEvC,KAAKU,6BAA6B;EAEpC,KAAKG,yBAAyB,MAAM,EAAE;EAEtC,IAAI,KAAKD,SAAS,UAAU,oBAC1B,KAAKF,6BAA6B;OAC7B,IAAI,CAAC,KAAKI,mBAAmB,KAAKF,SAAS,SAAS,GAAG;GAC5D,KAAKE,kBAAkB;GACvB,KAAKC,oBAAoB,mBACvB,KAAKL,6BAA6B,CACpC;EACF;CACF;CAEA,WAAW,QAAgB;EACzB,KAAKA,6BAA6B;EAClC,KAAK,MAAM,OAAO,KAAKX,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,KAAKa,SAAS,KAAK,KAAK;EAExB,MAAM,QAAQ,KAAKI,cAAc;EACjC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKpB,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKY,yBAAyB,MAAM,EAAE;EAGtC,KAAKN,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,gBAAqC;EACnC,IAAI,KAAKQ,sBAAsB,KAAA,GAAW;GACxC,eAAe,KAAKA,iBAAiB;GACrC,KAAKA,oBAAoB,KAAA;EAC3B;EACA,KAAKD,kBAAkB;EACvB,IAAI,KAAKF,SAAS,WAAW,GAC3B,OAAO,CAAC;EAEV,MAAM,QAAQ,KAAKA;EACnB,KAAKA,WAAW,CAAC;EACjB,OAAO;CACT;CAEA,+BAA+B;EAC7B,MAAM,QAAQ,KAAKI,cAAc;EACjC,IAAI,MAAM,SAAS,GACjB,UAAU,gBAAgB,KAAKjB,QAAQ,OAAO,GAAG,KAAK;CAE1D;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKA,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKW,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKX,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}
1	+ {"version":3,"file":"forwarder.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n getOrCreateCounter,\n getOrCreateGauge,\n getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n flowControlConsensusTimeoutProportion: number;\n flowControlSlowSubscriberGracePeriodMs?: number \| undefined;\n // Injectable timers, for deterministic testing.\n setTimeoutFn?: typeof setTimeout \| undefined;\n clearTimeoutFn?: typeof clearTimeout \| undefined;\n};\n\nconst FORWARD_BATCH_SIZE = 64;\n\nexport class Forwarder {\n readonly #lc: LogContext;\n readonly #setTimeout: typeof setTimeout;\n readonly #clearTimeout: typeof clearTimeout;\n readonly #active = new Set<Subscriber>();\n readonly #queued = new Set<Subscriber>();\n readonly #earlyReleaseOptions: EarlyReleaseOptions \| undefined;\n readonly #flowControlSlowSubscriberGracePeriodMs: number \| undefined;\n\n readonly #flowControlWaits = getOrCreateCounter(\n 'replication',\n 'flow_control.waits',\n 'Flow-control checkpoints completed, labeled by release mode.',\n );\n readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n 'replication',\n 'flow_control.wait_duration',\n 'Time replication waits at flow-control checkpoints.',\n );\n #inTransaction = false;\n\n #pending: WatermarkedChange[] = [];\n #flushScheduled = false;\n #flushImmediateId: NodeJS.Immediate \| undefined = undefined;\n\n #currentBroadcast: Broadcast \| undefined;\n #progressMonitor: NodeJS.Timeout \| undefined;\n\n // The backup-replicator is never disconnected as a laggard, so its lag\n // detections are logged (rather than acted upon). Track the running total and\n // throttle the warning to at most once per grace period to avoid spamming the\n // log for the duration of a slow catchup.\n #backupLagEventCount = 0;\n #lastBackupLagWarnMs: number \| undefined;\n\n constructor(\n lc: LogContext,\n opts: ProgressMonitorOptions = {\n flowControlConsensusTimeoutProportion: 2.0,\n },\n ) {\n this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n this.#earlyReleaseOptions =\n opts.flowControlConsensusTimeoutProportion < 0\n ? undefined\n : {\n consensusTimeoutProportion:\n opts.flowControlConsensusTimeoutProportion,\n setTimeoutFn: this.#setTimeout,\n clearTimeoutFn: this.#clearTimeout,\n };\n this.#flowControlSlowSubscriberGracePeriodMs =\n opts.flowControlSlowSubscriberGracePeriodMs;\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.active_subscribers',\n 'Active change-stream subscribers receiving live changes.',\n ).addCallback(result => result.observe(this.#active.size));\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.queued_subscribers',\n 'Change-stream subscribers waiting for the current transaction to finish before activation.',\n ).addCallback(result => result.observe(this.#queued.size));\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.pending_messages',\n 'Downstream change-stream messages not yet acked by subscribers.',\n ).addCallback(result =>\n result.observe(this.#subscriberStats().pendingMessages),\n );\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.backlog_messages',\n 'Live change-stream messages buffered while subscribers catch up.',\n ).addCallback(result =>\n result.observe(this.#subscriberStats().backlogMessages),\n );\n\n getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n description:\n 'Live change-stream bytes buffered while subscribers catch up.',\n unit: 'By',\n }).addCallback(result =>\n result.observe(this.#subscriberStats().backlogBytes),\n );\n\n getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n description:\n 'Maximum live change-stream bytes buffered by a single subscriber.',\n unit: 'By',\n }).addCallback(result =>\n result.observe(this.#subscriberStats().maxBacklogBytes),\n );\n }\n\n startProgressMonitor() {\n clearInterval(this.#progressMonitor);\n this.#progressMonitor = setInterval(\n () => this.checkSubscriberProgress(performance.now()),\n 1000,\n );\n }\n\n /*\n Runs one progress-monitor tick: samples each active subscriber's change\n * rate, logs slow broadcasts, and disconnects subscribers that have been\n * continuously lagging for longer than the configured grace period.\n \n Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n * than private) so tests can drive a single tick with an explicit `now`.\n /\n checkSubscriberProgress(now: number) {\n let timedOut: Map<Subscriber, SubscriberStats> \| undefined;\n let slowestOnTime: SubscriberStats \| undefined;\n\n for (const sub of this.#active) {\n const stats = sub.sampleProcessRate(now).getStats();\n if (stats.missedLastTimeout) {\n (timedOut ??= new Map()).set(sub, stats);\n } else if (\n stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n ) {\n slowestOnTime = stats;\n }\n }\n this.#currentBroadcast?.logSlowRequests(now);\n\n if (\n this.#flowControlSlowSubscriberGracePeriodMs &&\n timedOut?.size &&\n slowestOnTime\n ) {\n for (const [sub, stats] of timedOut.entries()) {\n const rate =\n stats.processRate < slowestOnTime.processRate\n ? 'lagging'\n : 'catching-up';\n const laggingDuration = sub.reportChangeRate(now, rate);\n if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n if (sub.mode === 'backup') {\n // The backup-replicator can only be detected as a laggard during\n // initial catchup; after that it is required for consensus. If it\n // triggers laggard detection in the former case, just log and\n // continue; as disconnecting it would otherwise kill the\n // change-streamer task altogether.\n this.#backupLagEventCount++;\n const grace = this.#flowControlSlowSubscriberGracePeriodMs;\n if (\n this.#lastBackupLagWarnMs === undefined \|\|\n now - this.#lastBackupLagWarnMs >= grace\n ) {\n this.#lastBackupLagWarnMs = now;\n this.#lc.warn?.(\n `backup subscriber ${sub.id} has lagged for ${laggingDuration}ms ` +\n `(${this.#backupLagEventCount} lag events so far). ` +\n `this is only expected during initial catchup`,\n {stats},\n );\n }\n } else {\n this.#lc.warn?.(\n `serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n {stats},\n );\n sub.close(\n StreamTooFarBehind,\n `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n );\n }\n }\n }\n }\n }\n\n stopProgressMonitor() {\n clearInterval(this.#progressMonitor);\n this.#flushPendingWithoutTracking();\n }\n\n /\n `add()` is called in lock step with `Storer.catchup()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n /\n add(sub: Subscriber) {\n this.#flushPendingWithoutTracking();\n if (this.#inTransaction) {\n this.#queued.add(sub);\n } else {\n this.#active.add(sub);\n }\n }\n\n remove(sub: Subscriber) {\n this.#flushPendingWithoutTracking();\n this.#active.delete(sub);\n this.#queued.delete(sub);\n sub.close();\n }\n\n /\n `forward()` is called in lockstep with `Storer.store()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n \n This version of forward buffers changes across the current event loop turn\n * and dispatches them in batches. The change-streamer should call and await\n * {@link forwardWithFlowControl()} occasionally to avoid memory blowup.\n /\n forward(entry: WatermarkedChange) {\n this.#pending.push(entry);\n\n if (\n this.#queued.size > 0 &&\n (entry[1] === 'commit' \|\| entry[1] === 'rollback')\n ) {\n this.#flushPendingWithoutTracking();\n }\n this.#updateActiveSubscribers(entry[1]);\n\n if (this.#pending.length >= FORWARD_BATCH_SIZE) {\n this.#flushPendingWithoutTracking();\n } else if (!this.#flushScheduled && this.#pending.length > 0) {\n this.#flushScheduled = true;\n this.#flushImmediateId = setImmediate(() =>\n this.#flushPendingWithoutTracking(),\n );\n }\n }\n\n sendStatus(status: Status) {\n this.#flushPendingWithoutTracking();\n for (const sub of this.#active.values()) {\n sub.sendStatus(status);\n }\n }\n\n /\n The flow-control-aware equivalent of {@link forward()}, returning a\n * Promise that resolves when replication should continue.\n */\n async forwardWithFlowControl(entry: WatermarkedChange) {\n this.#pending.push(entry);\n\n const batch = this.#drainPending();\n const start = performance.now();\n const broadcast = new Broadcast(\n this.#lc,\n this.#active.values(),\n batch,\n this.#earlyReleaseOptions,\n );\n this.#updateActiveSubscribers(entry[1]);\n\n // set for progress tracking\n this.#currentBroadcast = broadcast;\n\n try {\n await broadcast.done;\n const attributes = {'release.mode': broadcast.releaseMode};\n this.#flowControlWaits.add(1, attributes);\n this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n } finally {\n // Technically #currentBroadcast may have changed, so only\n // unset if it if is still the same.\n if (this.#currentBroadcast === broadcast) {\n this.#currentBroadcast = undefined;\n }\n }\n }\n\n #drainPending(): WatermarkedChange[] {\n if (this.#flushImmediateId !== undefined) {\n clearImmediate(this.#flushImmediateId);\n this.#flushImmediateId = undefined;\n }\n this.#flushScheduled = false;\n if (this.#pending.length === 0) {\n return [];\n }\n const batch = this.#pending;\n this.#pending = [];\n return batch;\n }\n\n #flushPendingWithoutTracking() {\n const batch = this.#drainPending();\n if (batch.length > 0) {\n Broadcast.withoutTracking(this.#active.values(), batch);\n }\n }\n\n #subscriberStats() {\n let pendingMessages = 0;\n let backlogMessages = 0;\n let backlogBytes = 0;\n let maxBacklogBytes = 0;\n\n for (const sub of joinIterables(this.#active, this.#queued)) {\n const stats = sub.getStats();\n pendingMessages += stats.pending;\n backlogMessages += stats.backlog;\n backlogBytes += stats.backlogBytes;\n maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n }\n\n return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n }\n\n #updateActiveSubscribers(tag: ChangeTag) {\n switch (tag) {\n case 'begin':\n // While in a Transaction, all added subscribers are \"queued\" so that no\n // messages are forwarded to them. This state corresponds to being queued\n // for catchup in the Storer, which will retrieve historic changes\n // and call catchup() once the current transaction is committed.\n this.#inTransaction = true;\n break;\n case 'commit':\n case 'rollback':\n // Upon commit or rollback, all queued subscribers are transferred to\n // the active set. This means that they can receive messages starting\n // from the next transaction.\n //\n // Note that if catchup is still in progress (in the Storer), these messages\n // will be buffered in the backlog until catchup completes.\n this.#inTransaction = false;\n for (const sub of this.#queued.values()) {\n this.#active.add(sub);\n }\n this.#queued.clear();\n break;\n }\n }\n\n getAcks(): Set<string> {\n return new Set(\n joinIterables(\n wrapIterable(this.#active).map(s => s.acked),\n wrapIterable(this.#queued).map(s => s.acked),\n ),\n );\n }\n}\n"],"mappings":";;;;AAoBA,IAAM,qBAAqB;AAE3B,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB,WAAgC,CAAC;CACjC,kBAAkB;CAClB,oBAAkD,KAAA;CAElD;CACA;CAMA,uBAAuB;CACvB;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAK,cAAc,KAAK,gBAAgB;EACxC,KAAK,gBAAgB,KAAK,kBAAkB;EAE5C,KAAK,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAK;GACnB,gBAAgB,KAAK;EACvB;EACN,KAAK,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,CAAC,CAAC,aAAY,WAAU,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,CAAC,CAAC,aAAY,WAAU,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,CAAC,CAAC,aAAY,WACZ,OAAO,QAAQ,KAAK,iBAAiB,CAAC,CAAC,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,CAAC,CAAC,aAAY,WACZ,OAAO,QAAQ,KAAK,iBAAiB,CAAC,CAAC,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,WACb,OAAO,QAAQ,KAAK,iBAAiB,CAAC,CAAC,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,WACb,OAAO,QAAQ,KAAK,iBAAiB,CAAC,CAAC,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAK,gBAAgB;EACnC,KAAK,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAK,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,CAAC,CAAC,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,EAAA,CAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAK,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAK,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAK,yCAAyC;IACnE,IAAI,IAAI,SAAS,UAAU;KAMzB,KAAK;KACL,MAAM,QAAQ,KAAK;KACnB,IACE,KAAK,yBAAyB,KAAA,KAC9B,MAAM,KAAK,wBAAwB,OACnC;MACA,KAAK,uBAAuB;MAC5B,KAAK,IAAI,OACP,qBAAqB,IAAI,GAAG,kBAAkB,gBAAgB,MACxD,KAAK,qBAAqB,oEAEhC,EAAC,MAAK,CACR;KACF;IACF,OAAO;KACL,KAAK,IAAI,OACP,sBAAsB,IAAI,GAAG,kBAAkB,gBAAgB,oBAC/D,EAAC,MAAK,CACR;KACA,IAAI,MAAA,GAEF,oBAAoB,KAAK,wCAAwC,MAAM,gBAAgB,IACzF;IACF;GACF;EACF;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAK,gBAAgB;EACnC,KAAK,6BAA6B;CACpC;;;;;;CAOA,IAAI,KAAiB;EACnB,KAAK,6BAA6B;EAClC,IAAI,KAAK,gBACP,KAAK,QAAQ,IAAI,GAAG;OAEpB,KAAK,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAK,6BAA6B;EAClC,KAAK,QAAQ,OAAO,GAAG;EACvB,KAAK,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,KAAK,SAAS,KAAK,KAAK;EAExB,IACE,KAAK,QAAQ,OAAO,MACnB,MAAM,OAAO,YAAY,MAAM,OAAO,aAEvC,KAAK,6BAA6B;EAEpC,KAAK,yBAAyB,MAAM,EAAE;EAEtC,IAAI,KAAK,SAAS,UAAU,oBAC1B,KAAK,6BAA6B;OAC7B,IAAI,CAAC,KAAK,mBAAmB,KAAK,SAAS,SAAS,GAAG;GAC5D,KAAK,kBAAkB;GACvB,KAAK,oBAAoB,mBACvB,KAAK,6BAA6B,CACpC;EACF;CACF;CAEA,WAAW,QAAgB;EACzB,KAAK,6BAA6B;EAClC,KAAK,MAAM,OAAO,KAAK,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,KAAK,SAAS,KAAK,KAAK;EAExB,MAAM,QAAQ,KAAK,cAAc;EACjC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAK,KACL,KAAK,QAAQ,OAAO,GACpB,OACA,KAAK,oBACP;EACA,KAAK,yBAAyB,MAAM,EAAE;EAGtC,KAAK,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAK,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAK,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAK,sBAAsB,WAC7B,KAAK,oBAAoB,KAAA;EAE7B;CACF;CAEA,gBAAqC;EACnC,IAAI,KAAK,sBAAsB,KAAA,GAAW;GACxC,eAAe,KAAK,iBAAiB;GACrC,KAAK,oBAAoB,KAAA;EAC3B;EACA,KAAK,kBAAkB;EACvB,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO,CAAC;EAEV,MAAM,QAAQ,KAAK;EACnB,KAAK,WAAW,CAAC;EACjB,OAAO;CACT;CAEA,+BAA+B;EAC7B,MAAM,QAAQ,KAAK,cAAc;EACjC,IAAI,MAAM,SAAS,GACjB,UAAU,gBAAgB,KAAK,QAAQ,OAAO,GAAG,KAAK;CAE1D;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAK,SAAS,KAAK,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAK,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAK,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAK,QAAQ,OAAO,GACpC,KAAK,QAAQ,IAAI,GAAG;IAEtB,KAAK,QAAQ,MAAM;EAEvB;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAK,OAAO,CAAC,CAAC,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAK,OAAO,CAAC,CAAC,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.js CHANGED Viewed

@@ -16,7 +16,7 @@ var SERVER_TIMEOUT_SECONDS = 60;
 * Client-side timeout, slightly above the server timeout so that litestream,
 * rather than the client, ends a slow request.
 */
-var CLIENT_TIMEOUT_MS = (SERVER_TIMEOUT_SECONDS + 5) * 1e3;
+var CLIENT_TIMEOUT_MS = 65e3;
 var ERROR_LOG_INTERVAL_MS = 6e4;
 /**
 * Initial delay before retrying a failed request, doubled on each consecutive

package/out/zero-cache/src/services/change-streamer/litestream-sync-requester.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"litestream-sync-requester.js","names":["#lc","#state","#config","#litestream","#readLocalWatermark","#stream","#timer","#retryTimer","#inFlight","#requestSync","#retryDelayMs","#publish","#scheduleRetry","#lastErrorLogMs","#backedUp"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream-sync-requester.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database, type Statement} from '../../../../zqlite/src/db.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {LitestreamController} from '../litestream/litestream-controller.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\n/*\n How long litestream waits for a requested sync (and its upload) before\n * ending the request. A request that times out is simply retried on a later\n * tick: litestream only abandons the wait, never the sync or upload itself,\n * so this bounds request lifetime without affecting backup progress.\n /\nconst SERVER_TIMEOUT_SECONDS = 60;\n\n/\n Client-side timeout, slightly above the server timeout so that litestream,\n * rather than the client, ends a slow request.\n /\nconst CLIENT_TIMEOUT_MS = (SERVER_TIMEOUT_SECONDS + 5) 1000;\n\nconst ERROR_LOG_INTERVAL_MS = 60_000;\n\n/*\n Initial delay before retrying a failed request, doubled on each consecutive\n * failure until it reaches the configured interval, after which the regular\n * ticks alone retry, so that a busy litestream sees no more requests than the\n * interval allows. The first request is typically issued before litestream's\n * control socket is listening; retrying quickly (rather than after a full\n * interval) keeps the initial backup confirmation, which gates readiness, from\n * waiting out an interval.\n /\nconst MIN_RETRY_DELAY_MS = 250;\n\nconst localWatermarkSchema = v.object({\n stateVersion: v.string(),\n writeTimeMs: v.number(),\n});\n\ntype LocalWatermark = v.Infer<typeof localWatermarkSchema>;\n\n/* The subset of {@link LitestreamController} used by the requester. /\nexport type SyncController = Pick<LitestreamController, 'sync' \| 'close'>;\n\nexport type SyncRequesterConfig = {\n /\n Interval between sync requests. This paces how often litestream seals\n * (and uploads) a new LTX file.\n /\n intervalMs: number;\n};\n\n/\n The LitestreamSyncRequester drives litestream v5 backups and tracks which\n * replica watermark is durably backed up, emitting it as a stream of\n * {@link BackedUpWatermark}s.\n \n It periodically reads the replica's current watermark `W` and then asks\n * litestream to sync and wait for the upload (`POST /sync {wait: true}`).\n * litestream satisfies the request with a sync pass that starts after the\n * request, and a pass syncs up to the end of the WAL, so a successful response\n * means that every transaction committed before the request, including the\n * one that wrote `W`, has been uploaded to the backup. No part of the backup\n * needs to be read to determine this.\n \n Requests continue even when `W` is already backed up, so that changes that\n * don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)\n * are also backed up at this cadence rather than only at litestream's slower\n * backstop interval. A request on an idle replica is a local no-op for\n * litestream: there is nothing to seal or upload, and no backup reads.\n \n At most one request is outstanding at a time. Failed or timed-out requests\n * are retried with a backoff that starts short and, once it grows to the\n * interval, defers to the regular ticks. Nothing is published until a request\n * succeeds.\n /\nexport class LitestreamSyncRequester {\n readonly #lc: LogContext;\n readonly #state = new RunningState('litestream-sync-requester');\n readonly #config: SyncRequesterConfig;\n readonly #litestream: SyncController;\n readonly #readLocalWatermark: Statement;\n readonly #stream: Subscription<BackedUpWatermark>;\n\n #timer: NodeJS.Timeout \| undefined;\n #retryTimer: NodeJS.Timeout \| undefined;\n #retryDelayMs = MIN_RETRY_DELAY_MS;\n #inFlight: Promise<void> \| undefined;\n #backedUp: string \| undefined;\n #lastErrorLogMs = 0;\n\n constructor(\n lc: LogContext,\n replicaFile: string,\n config: SyncRequesterConfig,\n litestream: SyncController = new LitestreamController(lc, replicaFile),\n ) {\n this.#lc = lc.withContext('component', 'litestream-sync-requester');\n this.#config = config;\n this.#litestream = litestream;\n\n const db = new Database(this.#lc, replicaFile, {readonly: true});\n this.#readLocalWatermark = db.prepare(\n `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n );\n this.#stream = Subscription.create<BackedUpWatermark>({\n cleanup: () => {\n clearInterval(this.#timer);\n clearTimeout(this.#retryTimer);\n this.#state.stop(this.#lc);\n this.#litestream.close();\n db.close();\n },\n });\n }\n\n /\n Starts requesting syncs and pushes backed-up watermarks to the returned\n * stream. Must only be called once.\n /\n start(): Source<BackedUpWatermark> {\n assert(this.#timer === undefined, `Already called start()`);\n this.#lc.info?.(\n `requesting litestream syncs every ${this.#config.intervalMs} ms`,\n );\n this.#timer = setInterval(this.tick, this.#config.intervalMs);\n void this.tick();\n return this.#stream;\n }\n\n /\n Requests a sync unless a request is already outstanding. Resolves when\n * this tick's request (or the outstanding one) completes.\n * Exported for testing.\n /\n readonly tick = (): Promise<void> => {\n if (!this.#state.shouldRun() \|\| this.#inFlight) {\n return this.#inFlight ?? Promise.resolve();\n }\n let local: LocalWatermark;\n try {\n local = v.parse(this.#readLocalWatermark.get(), localWatermarkSchema);\n } catch (e) {\n this.#lc.warn?.(`unable to read local watermark`, e);\n return Promise.resolve();\n }\n this.#inFlight = this.#requestSync(local).finally(() => {\n this.#inFlight = undefined;\n });\n return this.#inFlight;\n };\n\n async #requestSync(local: LocalWatermark) {\n // `local` must be read before the request is issued: litestream only\n // guarantees that transactions committed before the request are synced.\n try {\n const result = await this.#litestream.sync(\n {\n wait: true,\n timeoutMs: CLIENT_TIMEOUT_MS,\n serverTimeoutSeconds: SERVER_TIMEOUT_SECONDS,\n },\n this.#state.signal,\n );\n this.#retryDelayMs = MIN_RETRY_DELAY_MS;\n this.#publish(local, Date.now());\n this.#lc.debug?.(`backed up watermark ${local.stateVersion}`, result);\n } catch (e) {\n if (!this.#state.shouldRun()) {\n return; // shutting down\n }\n // Expected while litestream is starting up or busy with a long sync or\n // upload (e.g. the initial snapshot of a large replica); the request is\n // retried after a backoff, or on the next tick if that comes first.\n this.#scheduleRetry();\n const now = Date.now();\n if (now - this.#lastErrorLogMs >= ERROR_LOG_INTERVAL_MS) {\n this.#lastErrorLogMs = now;\n this.#lc.info?.(\n `backup of watermark ${local.stateVersion} not yet confirmed. retrying`,\n e,\n );\n }\n }\n }\n\n #scheduleRetry() {\n clearTimeout(this.#retryTimer);\n const delayMs = this.#retryDelayMs;\n if (delayMs >= this.#config.intervalMs) {\n return; // backed off to the interval; the regular ticks retry\n }\n this.#retryDelayMs = delayMs 2;\n this.#retryTimer = setTimeout(this.tick, delayMs);\n }\n\n #publish({stateVersion, writeTimeMs}: LocalWatermark, backupTimeMs: number) {\n if (this.#backedUp !== undefined && stateVersion <= this.#backedUp) {\n return;\n }\n this.#backedUp = stateVersion;\n this.#stream.push({watermark: stateVersion, writeTimeMs, backupTimeMs});\n }\n}\n"],"mappings":";;;;;;;;;;;;;AAgBA,IAAM,yBAAyB;;;;;AAM/B,IAAM,qBAAqB,yBAAyB,KAAK;AAEzD,IAAM,wBAAwB;;;;;;;;;;AAW9B,IAAM,qBAAqB;AAE3B,IAAM,uBAAuB,eAAE,OAAO;CACpC,cAAc,eAAE,OAAO;CACvB,aAAa,eAAE,OAAO;AACxB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;AAuCD,IAAa,0BAAb,MAAqC;CACnC;CACA,SAAkB,IAAI,aAAa,2BAA2B;CAC9D;CACA;CACA;CACA;CAEA;CACA;CACA,gBAAgB;CAChB;CACA;CACA,kBAAkB;CAElB,YACE,IACA,aACA,QACA,aAA6B,IAAI,qBAAqB,IAAI,WAAW,GACrE;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAKE,UAAU;EACf,KAAKC,cAAc;EAEnB,MAAM,KAAK,IAAI,SAAS,KAAKH,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,KAAKI,sBAAsB,GAAG,QAC5B,gEACF;EACA,KAAKC,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAKC,MAAM;GACzB,aAAa,KAAKC,WAAW;GAC7B,KAAKN,OAAO,KAAK,KAAKD,GAAG;GACzB,KAAKG,YAAY,MAAM;GACvB,GAAG,MAAM;EACX,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKG,WAAW,KAAA,GAAW,wBAAwB;EAC1D,KAAKN,IAAI,OACP,qCAAqC,KAAKE,QAAQ,WAAW,IAC/D;EACA,KAAKI,SAAS,YAAY,KAAK,MAAM,KAAKJ,QAAQ,UAAU;EAC5D,KAAU,KAAK;EACf,OAAO,KAAKG;CACd;;;;;;CAOA,aAAqC;EACnC,IAAI,CAAC,KAAKJ,OAAO,UAAU,KAAK,KAAKO,WACnC,OAAO,KAAKA,aAAa,QAAQ,QAAQ;EAE3C,IAAI;EACJ,IAAI;GACF,QAAQ,MAAQ,KAAKJ,oBAAoB,IAAI,GAAG,oBAAoB;EACtE,SAAS,GAAG;GACV,KAAKJ,IAAI,OAAO,kCAAkC,CAAC;GACnD,OAAO,QAAQ,QAAQ;EACzB;EACA,KAAKQ,YAAY,KAAKC,aAAa,KAAK,EAAE,cAAc;GACtD,KAAKD,YAAY,KAAA;EACnB,CAAC;EACD,OAAO,KAAKA;CACd;CAEA,MAAMC,aAAa,OAAuB;EAGxC,IAAI;GACF,MAAM,SAAS,MAAM,KAAKN,YAAY,KACpC;IACE,MAAM;IACN,WAAW;IACX,sBAAsB;GACxB,GACA,KAAKF,OAAO,MACd;GACA,KAAKS,gBAAgB;GACrB,KAAKC,SAAS,OAAO,KAAK,IAAI,CAAC;GAC/B,KAAKX,IAAI,QAAQ,uBAAuB,MAAM,gBAAgB,MAAM;EACtE,SAAS,GAAG;GACV,IAAI,CAAC,KAAKC,OAAO,UAAU,GACzB;GAKF,KAAKW,eAAe;GACpB,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,MAAM,KAAKC,mBAAmB,uBAAuB;IACvD,KAAKA,kBAAkB;IACvB,KAAKb,IAAI,OACP,uBAAuB,MAAM,aAAa,+BAC1C,CACF;GACF;EACF;CACF;CAEA,iBAAiB;EACf,aAAa,KAAKO,WAAW;EAC7B,MAAM,UAAU,KAAKG;EACrB,IAAI,WAAW,KAAKR,QAAQ,YAC1B;EAEF,KAAKQ,gBAAgB,UAAU;EAC/B,KAAKH,cAAc,WAAW,KAAK,MAAM,OAAO;CAClD;CAEA,SAAS,EAAC,cAAc,eAA8B,cAAsB;EAC1E,IAAI,KAAKO,cAAc,KAAA,KAAa,gBAAgB,KAAKA,WACvD;EAEF,KAAKA,YAAY;EACjB,KAAKT,QAAQ,KAAK;GAAC,WAAW;GAAc;GAAa;EAAY,CAAC;CACxE;AACF"}
1	+ {"version":3,"file":"litestream-sync-requester.js","names":["v","v.parse"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream-sync-requester.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database, type Statement} from '../../../../zqlite/src/db.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {LitestreamController} from '../litestream/litestream-controller.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\n/*\n How long litestream waits for a requested sync (and its upload) before\n * ending the request. A request that times out is simply retried on a later\n * tick: litestream only abandons the wait, never the sync or upload itself,\n * so this bounds request lifetime without affecting backup progress.\n /\nconst SERVER_TIMEOUT_SECONDS = 60;\n\n/\n Client-side timeout, slightly above the server timeout so that litestream,\n * rather than the client, ends a slow request.\n /\nconst CLIENT_TIMEOUT_MS = (SERVER_TIMEOUT_SECONDS + 5) 1000;\n\nconst ERROR_LOG_INTERVAL_MS = 60_000;\n\n/*\n Initial delay before retrying a failed request, doubled on each consecutive\n * failure until it reaches the configured interval, after which the regular\n * ticks alone retry, so that a busy litestream sees no more requests than the\n * interval allows. The first request is typically issued before litestream's\n * control socket is listening; retrying quickly (rather than after a full\n * interval) keeps the initial backup confirmation, which gates readiness, from\n * waiting out an interval.\n /\nconst MIN_RETRY_DELAY_MS = 250;\n\nconst localWatermarkSchema = v.object({\n stateVersion: v.string(),\n writeTimeMs: v.number(),\n});\n\ntype LocalWatermark = v.Infer<typeof localWatermarkSchema>;\n\n/* The subset of {@link LitestreamController} used by the requester. /\nexport type SyncController = Pick<LitestreamController, 'sync' \| 'close'>;\n\nexport type SyncRequesterConfig = {\n /\n Interval between sync requests. This paces how often litestream seals\n * (and uploads) a new LTX file.\n /\n intervalMs: number;\n};\n\n/\n The LitestreamSyncRequester drives litestream v5 backups and tracks which\n * replica watermark is durably backed up, emitting it as a stream of\n * {@link BackedUpWatermark}s.\n \n It periodically reads the replica's current watermark `W` and then asks\n * litestream to sync and wait for the upload (`POST /sync {wait: true}`).\n * litestream satisfies the request with a sync pass that starts after the\n * request, and a pass syncs up to the end of the WAL, so a successful response\n * means that every transaction committed before the request, including the\n * one that wrote `W`, has been uploaded to the backup. No part of the backup\n * needs to be read to determine this.\n \n Requests continue even when `W` is already backed up, so that changes that\n * don't advance the watermark (e.g. change-log cleanup or `PRAGMA optimize`)\n * are also backed up at this cadence rather than only at litestream's slower\n * backstop interval. A request on an idle replica is a local no-op for\n * litestream: there is nothing to seal or upload, and no backup reads.\n \n At most one request is outstanding at a time. Failed or timed-out requests\n * are retried with a backoff that starts short and, once it grows to the\n * interval, defers to the regular ticks. Nothing is published until a request\n * succeeds.\n /\nexport class LitestreamSyncRequester {\n readonly #lc: LogContext;\n readonly #state = new RunningState('litestream-sync-requester');\n readonly #config: SyncRequesterConfig;\n readonly #litestream: SyncController;\n readonly #readLocalWatermark: Statement;\n readonly #stream: Subscription<BackedUpWatermark>;\n\n #timer: NodeJS.Timeout \| undefined;\n #retryTimer: NodeJS.Timeout \| undefined;\n #retryDelayMs = MIN_RETRY_DELAY_MS;\n #inFlight: Promise<void> \| undefined;\n #backedUp: string \| undefined;\n #lastErrorLogMs = 0;\n\n constructor(\n lc: LogContext,\n replicaFile: string,\n config: SyncRequesterConfig,\n litestream: SyncController = new LitestreamController(lc, replicaFile),\n ) {\n this.#lc = lc.withContext('component', 'litestream-sync-requester');\n this.#config = config;\n this.#litestream = litestream;\n\n const db = new Database(this.#lc, replicaFile, {readonly: true});\n this.#readLocalWatermark = db.prepare(\n `SELECT stateVersion, writeTimeMs FROM \"_zero.replicationState\"`,\n );\n this.#stream = Subscription.create<BackedUpWatermark>({\n cleanup: () => {\n clearInterval(this.#timer);\n clearTimeout(this.#retryTimer);\n this.#state.stop(this.#lc);\n this.#litestream.close();\n db.close();\n },\n });\n }\n\n /\n Starts requesting syncs and pushes backed-up watermarks to the returned\n * stream. Must only be called once.\n /\n start(): Source<BackedUpWatermark> {\n assert(this.#timer === undefined, `Already called start()`);\n this.#lc.info?.(\n `requesting litestream syncs every ${this.#config.intervalMs} ms`,\n );\n this.#timer = setInterval(this.tick, this.#config.intervalMs);\n void this.tick();\n return this.#stream;\n }\n\n /\n Requests a sync unless a request is already outstanding. Resolves when\n * this tick's request (or the outstanding one) completes.\n * Exported for testing.\n /\n readonly tick = (): Promise<void> => {\n if (!this.#state.shouldRun() \|\| this.#inFlight) {\n return this.#inFlight ?? Promise.resolve();\n }\n let local: LocalWatermark;\n try {\n local = v.parse(this.#readLocalWatermark.get(), localWatermarkSchema);\n } catch (e) {\n this.#lc.warn?.(`unable to read local watermark`, e);\n return Promise.resolve();\n }\n this.#inFlight = this.#requestSync(local).finally(() => {\n this.#inFlight = undefined;\n });\n return this.#inFlight;\n };\n\n async #requestSync(local: LocalWatermark) {\n // `local` must be read before the request is issued: litestream only\n // guarantees that transactions committed before the request are synced.\n try {\n const result = await this.#litestream.sync(\n {\n wait: true,\n timeoutMs: CLIENT_TIMEOUT_MS,\n serverTimeoutSeconds: SERVER_TIMEOUT_SECONDS,\n },\n this.#state.signal,\n );\n this.#retryDelayMs = MIN_RETRY_DELAY_MS;\n this.#publish(local, Date.now());\n this.#lc.debug?.(`backed up watermark ${local.stateVersion}`, result);\n } catch (e) {\n if (!this.#state.shouldRun()) {\n return; // shutting down\n }\n // Expected while litestream is starting up or busy with a long sync or\n // upload (e.g. the initial snapshot of a large replica); the request is\n // retried after a backoff, or on the next tick if that comes first.\n this.#scheduleRetry();\n const now = Date.now();\n if (now - this.#lastErrorLogMs >= ERROR_LOG_INTERVAL_MS) {\n this.#lastErrorLogMs = now;\n this.#lc.info?.(\n `backup of watermark ${local.stateVersion} not yet confirmed. retrying`,\n e,\n );\n }\n }\n }\n\n #scheduleRetry() {\n clearTimeout(this.#retryTimer);\n const delayMs = this.#retryDelayMs;\n if (delayMs >= this.#config.intervalMs) {\n return; // backed off to the interval; the regular ticks retry\n }\n this.#retryDelayMs = delayMs 2;\n this.#retryTimer = setTimeout(this.tick, delayMs);\n }\n\n #publish({stateVersion, writeTimeMs}: LocalWatermark, backupTimeMs: number) {\n if (this.#backedUp !== undefined && stateVersion <= this.#backedUp) {\n return;\n }\n this.#backedUp = stateVersion;\n this.#stream.push({watermark: stateVersion, writeTimeMs, backupTimeMs});\n }\n}\n"],"mappings":";;;;;;;;;;;;;AAgBA,IAAM,yBAAyB;;;;;AAM/B,IAAM,oBAAA;AAEN,IAAM,wBAAwB;;;;;;;;;;AAW9B,IAAM,qBAAqB;AAE3B,IAAM,uBAAuBA,eAAE,OAAO;CACpC,cAAcA,eAAE,OAAO;CACvB,aAAaA,eAAE,OAAO;AACxB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;AAuCD,IAAa,0BAAb,MAAqC;CACnC;CACA,SAAkB,IAAI,aAAa,2BAA2B;CAC9D;CACA;CACA;CACA;CAEA;CACA;CACA,gBAAgB;CAChB;CACA;CACA,kBAAkB;CAElB,YACE,IACA,aACA,QACA,aAA6B,IAAI,qBAAqB,IAAI,WAAW,GACrE;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAK,UAAU;EACf,KAAK,cAAc;EAEnB,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,KAAK,sBAAsB,GAAG,QAC5B,gEACF;EACA,KAAK,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAK,MAAM;GACzB,aAAa,KAAK,WAAW;GAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;GACzB,KAAK,YAAY,MAAM;GACvB,GAAG,MAAM;EACX,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAK,WAAW,KAAA,GAAW,wBAAwB;EAC1D,KAAK,IAAI,OACP,qCAAqC,KAAK,QAAQ,WAAW,IAC/D;EACA,KAAK,SAAS,YAAY,KAAK,MAAM,KAAK,QAAQ,UAAU;EAC5D,KAAU,KAAK;EACf,OAAO,KAAK;CACd;;;;;;CAOA,aAAqC;EACnC,IAAI,CAAC,KAAK,OAAO,UAAU,KAAK,KAAK,WACnC,OAAO,KAAK,aAAa,QAAQ,QAAQ;EAE3C,IAAI;EACJ,IAAI;GACF,QAAQC,MAAQ,KAAK,oBAAoB,IAAI,GAAG,oBAAoB;EACtE,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,kCAAkC,CAAC;GACnD,OAAO,QAAQ,QAAQ;EACzB;EACA,KAAK,YAAY,KAAK,aAAa,KAAK,CAAC,CAAC,cAAc;GACtD,KAAK,YAAY,KAAA;EACnB,CAAC;EACD,OAAO,KAAK;CACd;CAEA,MAAM,aAAa,OAAuB;EAGxC,IAAI;GACF,MAAM,SAAS,MAAM,KAAK,YAAY,KACpC;IACE,MAAM;IACN,WAAW;IACX,sBAAsB;GACxB,GACA,KAAK,OAAO,MACd;GACA,KAAK,gBAAgB;GACrB,KAAK,SAAS,OAAO,KAAK,IAAI,CAAC;GAC/B,KAAK,IAAI,QAAQ,uBAAuB,MAAM,gBAAgB,MAAM;EACtE,SAAS,GAAG;GACV,IAAI,CAAC,KAAK,OAAO,UAAU,GACzB;GAKF,KAAK,eAAe;GACpB,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,MAAM,KAAK,mBAAmB,uBAAuB;IACvD,KAAK,kBAAkB;IACvB,KAAK,IAAI,OACP,uBAAuB,MAAM,aAAa,+BAC1C,CACF;GACF;EACF;CACF;CAEA,iBAAiB;EACf,aAAa,KAAK,WAAW;EAC7B,MAAM,UAAU,KAAK;EACrB,IAAI,WAAW,KAAK,QAAQ,YAC1B;EAEF,KAAK,gBAAgB,UAAU;EAC/B,KAAK,cAAc,WAAW,KAAK,MAAM,OAAO;CAClD;CAEA,SAAS,EAAC,cAAc,eAA8B,cAAsB;EAC1E,IAAI,KAAK,cAAc,KAAA,KAAa,gBAAgB,KAAK,WACvD;EAEF,KAAK,YAAY;EACjB,KAAK,QAAQ,KAAK;GAAC,WAAW;GAAc;GAAa;EAAY,CAAC;CACxE;AACF"}

package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js CHANGED Viewed

@@ -35,7 +35,7 @@ var BACKUP_VERIFICATION_SLACK_MS = 6e4;
 * staleness clock is reset the moment a purge is confirmed against a real
 * upload.
 */
-var WEDGED_SHUTDOWN_GRACE_MS = 15 * 6e4;
+var WEDGED_SHUTDOWN_GRACE_MS = 9e5;
 var GiB = 1024 ** 3;
 /**
 * How long the BackupMonitor waits for the *first* restorable backup to appear
@@ -51,7 +51,7 @@ var GiB = 1024 ** 3;
 * 50GB) so that it never fires before the platform would restart a view-syncer
 * that is waiting on the backup.
 */
-var INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 6e4;
+var INITIAL_BACKUP_GRACE_MS_PER_UNIT = 36e5;
 var INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB;
 /**
 * The Litestream3PrometheusPoller polls the litestream "/metrics" endpoint to
@@ -310,6 +310,6 @@ var Litestream3PrometheusPoller = class {
 	}
 };
 //#endregion
-export { Litestream3PrometheusPoller };
+export { BACKUP_VERIFICATION_SLACK_MS, CHECK_INTERVAL_MS, INITIAL_BACKUP_GRACE_MS_PER_UNIT, Litestream3PrometheusPoller, WEDGED_SHUTDOWN_GRACE_MS };
 //# sourceMappingURL=litestream3-prometheus-poller.js.map

package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"litestream3-prometheus-poller.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#state","#stream","#watermarks","#verifyBackupState","#purgesBlocked","#checkMetricsTimer","#runStartTime","#confirmRestorableBackup","#lastVerifiedUploadTime","#verifyBackupStateTimed","#checkWatermarks","#publishVerifiedWatermarks","#checkInitialBackupDeadline","#fetchWatermarks","#lastWatermark","#latestBackupTime","#maxBackupBefore","#confirmedDurable","#backupStaleSince","#shutDownOnWedgedBackup","#initialBackupGraceMs","#shutDownOnMissingInitialBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n litestreamBackupVerificationDuration,\n litestreamMonitorMetricAttrs,\n} from '../litestream/metrics.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nexport const CHECK_INTERVAL_MS = 10_000;\n\n/*\n Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n /\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/\n How long the actual backup state may remain continuously behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n \n A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes some such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n \n The change-log is conservatively not purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n /\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 60_000; // 15 minutes\n\nconst GiB = 1024 3;\n\n/\n * How long the BackupMonitor waits for the first restorable backup to appear\n * before concluding that the backup pipeline is broken and shutting the process\n * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).\n \n On a fresh stack the first backup is not restorable until the initial\n * Postgres->replica sync completes and litestream finishes uploading the\n * initial snapshot — a multipart upload that is not listable in the destination\n * until it is committed. The producer cannot observe that sub-snapshot progress,\n * so this is necessarily a generous timeout. It is scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance (~1 hour per\n * 50GB) so that it never fires before the platform would restart a view-syncer\n * that is waiting on the backup.\n /\nexport const INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 60_000; // 1 hour ...\nconst INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB; // ... per 50 GiB\n\n/*\n How often the BackupMonitor re-checks whether the first restorable backup has\n * appeared while a view-syncer holds a snapshot reservation open during a cold\n * start.\n /\nexport const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10_000;\n\n/\n Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n \n See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n /\nexport type BackupStateVerifier = () => Promise<Date>;\n\n/\n The Litestream3PrometheusPoller polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and emits verified watermarks to a stream of {@link BackedUpWatermark}s.\n \n See: https://github.com/rocicorp/litestream/pull/3\n \n Because the watermarks reported by litestream metrics reflect what\n * litestream believes has been backed up (which has been observed to\n * diverge from reality when uploads silently fail), the cleanup watermark is\n * only published after verifying it against the actual backup state in the\n * replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n /\nexport class Litestream3PrometheusPoller {\n readonly #lc: LogContext;\n readonly #replicaFile: string;\n readonly #backupURL: string;\n readonly #metricsEndpoint: string;\n readonly #state = new RunningState('litestream3-prometheus-poller');\n\n readonly #stream: Subscription<BackedUpWatermark>;\n readonly #watermarks = new Map<string, Date>();\n\n readonly #verifyBackupState: BackupStateVerifier;\n readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n description:\n 'Number of change-log purges blocked because the actual backup state ' +\n '(as listed from the replica destination) could not be verified, or ' +\n 'is older than the backup progress claimed by litestream metrics. ' +\n 'A steadily increasing value indicates a wedged or failing backup. ' +\n 'The \"backup-wedged\" reason is emitted once just before the process ' +\n 'shuts itself down due to a persistently stale backup.',\n });\n\n #lastWatermark: string = '';\n #latestBackupTime: Date \| null = null;\n // The time of the most recent object actually verified in the backup\n // destination, or `null` if none has ever been verified. Only assigned from a\n // successful verifyBackupState() and never cleared, so a non-null value also\n // means a restorable backup has been confirmed to exist (which is what gates\n // the snapshot `status` signal and satisfies the initial-backup deadline).\n #lastVerifiedUploadTime: Date \| null = null;\n // Epoch ms at which the actual backup state was first observed to be\n // continuously behind the watermark litestream claims to have backed up,\n // or `null` while the backup is keeping up. Reset whenever a purge is\n // confirmed against a real upload.\n #backupStaleSince: number \| null = null;\n // Epoch ms at which run() started, used to bound how long to wait for the\n // first restorable backup to appear. `null` until run() is called.\n #runStartTime: number \| null = null;\n #checkMetricsTimer: NodeJS.Timeout \| undefined;\n\n constructor(\n lc: LogContext,\n replicaFile: string,\n backupURL: string,\n metricsEndpoint: string,\n verifyBackupState: BackupStateVerifier,\n ) {\n this.#lc = lc.withContext('component', 'litestream3-prometheus-poller');\n this.#replicaFile = replicaFile;\n this.#backupURL = backupURL;\n this.#metricsEndpoint = metricsEndpoint;\n this.#verifyBackupState = verifyBackupState;\n this.#stream = Subscription.create<BackedUpWatermark>({\n cleanup: () => {\n clearInterval(this.#checkMetricsTimer);\n this.#state.stop(this.#lc);\n },\n });\n }\n\n /\n Starts polling for watermarks and pushes new ones to returned stream.\n * Must only be called once.\n /\n start(): Source<BackedUpWatermark> {\n assert(this.#runStartTime === null, `Already called start()`);\n this.#runStartTime = Date.now();\n this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint}`);\n this.#checkMetricsTimer = setInterval(\n this.checkVerifyAndPublishWatermarks,\n CHECK_INTERVAL_MS,\n );\n return this.#stream;\n }\n\n /\n Verifies whether a restorable backup actually exists in the destination\n * (as opposed to what litestream metrics merely claim). On success the\n * verified upload time is cached — seeding the cleanup-verification fast path\n * ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.\n * Returns false if no backup is listable yet (e.g. the initial snapshot is\n * still uploading).\n /\n async #confirmRestorableBackup(): Promise<boolean> {\n try {\n this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n return true;\n } catch (e) {\n this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);\n return false;\n }\n }\n\n // Exported for testing\n readonly checkVerifyAndPublishWatermarks = async () => {\n try {\n await this.#checkWatermarks();\n } catch (e) {\n this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n }\n try {\n await this.#publishVerifiedWatermarks();\n } catch (e) {\n this.#lc.warn?.(`error scheduling cleanup`, e);\n }\n try {\n await this.#checkInitialBackupDeadline();\n } catch (e) {\n this.#lc.warn?.(`error checking initial backup deadline`, e);\n }\n };\n\n async #fetchWatermarks(): AsyncGenerator<{\n watermark: string;\n time: Date;\n name?: string \| undefined;\n }> {\n const metricsEndpoint = this.#metricsEndpoint;\n const signal = this.#state.signal;\n let resp;\n try {\n resp = await fetch(metricsEndpoint, {signal});\n } catch (e) {\n if (signal.aborted) {\n // not an error.\n return;\n }\n // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n // log them and skip the watermark check until the next interval.\n this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n return;\n }\n if (!resp.ok) {\n this.#lc.warn?.(\n `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n );\n return;\n }\n\n const families = parsePrometheusTextFormat(await resp.text());\n for (const family of families) {\n if (\n family.type === 'GAUGE' &&\n family.name === 'litestream_replica_progress'\n ) {\n for (const metric of family.metrics) {\n const watermark = metric.labels?.watermark;\n const name = metric.labels?.name;\n const time = new Date(parseFloat(metric.value) * 1000);\n if (watermark) {\n try {\n versionFromLexi(watermark); // Sanity check: Ensure that it is valid.\n yield {watermark, time, name};\n } catch (e) {\n this.#lc.warn?.(\n `ignoring invalid watermark ${watermark}`,\n {metric},\n e,\n );\n }\n }\n }\n }\n }\n }\n\n async #checkWatermarks() {\n for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n this.#lc.info?.(\n `replicated watermark=${watermark} to ${name}` +\n ` at ${time.toISOString()}.`,\n );\n this.#watermarks.set(watermark, time);\n this.#latestBackupTime = time;\n }\n }\n return this.#latestBackupTime;\n }\n\n async #publishVerifiedWatermarks() {\n const maxBackup = this.#maxBackupBefore(Date.now());\n if (!maxBackup) {\n return;\n }\n const {watermark: maxWatermark, backupTime: claimedTime} = maxBackup;\n // Purge guard: the watermarks (and their backup times) come from\n // litestream metrics, which are exported when litestream believes\n // an upload succeeded, and have been observed to advance even when\n // nothing was actually written to the backup destination. Purging the\n // change-log based on a falsely advancing watermark permanently breaks\n // the ability to restore + catch up. Before advancing the cleanup\n // watermark, verify it against the actual backup state: a claimed\n // backup time is only trusted if an object was actually uploaded to\n // the replica destination at (or after) that time, modulo clock skew.\n if (!this.#confirmedDurable(claimedTime)) {\n try {\n this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n } catch (e) {\n this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n // Skipping the purge is safe: the change-log just grows.\n this.#lc.warn?.(\n `unable to verify backup state. skipping change-log cleanup ` +\n `up to watermark ${maxWatermark} ` +\n `(claimed backup time ${claimedTime.toISOString()})`,\n e,\n );\n return;\n }\n }\n // Watermarks whose backup time isn't yet confirmed durable remain in the\n // map and are re-evaluated at the next check.\n const lastUpload = must(this.#lastVerifiedUploadTime);\n const verifiedBackup = this.#maxBackupBefore(\n lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n );\n if (!verifiedBackup) {\n this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n const now = Date.now();\n if (this.#backupStaleSince === null) {\n this.#backupStaleSince = now;\n }\n const staleForMs = now - this.#backupStaleSince;\n this.#lc.warn?.(\n `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n `litestream claims it was backed up at ` +\n `${claimedTime.toISOString()}, but the last object actually ` +\n `uploaded to ${this.#backupURL} was at ` +\n `${lastUpload.toISOString()}. ` +\n `The backup has been stale for ${staleForMs} ms.`,\n );\n if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n }\n return;\n }\n // The cleanup watermark advanced past a real upload, so the backup is\n // keeping up: clear the staleness clock.\n this.#backupStaleSince = null;\n const {watermark: verifiedWatermark, backupTime} = verifiedBackup;\n for (const watermark of this.#watermarks.keys()) {\n if (watermark <= verifiedWatermark) {\n this.#watermarks.delete(watermark);\n }\n }\n this.#stream.push({\n watermark: verifiedWatermark,\n backupTimeMs: backupTime.getTime(),\n });\n this.#lastWatermark = verifiedWatermark;\n }\n\n async #verifyBackupStateTimed(): Promise<Date> {\n const start = performance.now();\n let result: 'success' \| 'error' = 'error';\n try {\n const ret = await this.#verifyBackupState();\n result = 'success';\n return ret;\n } finally {\n litestreamBackupVerificationDuration().recordMs(\n performance.now() - start,\n {\n ...litestreamMonitorMetricAttrs(\n this.#backupURL,\n 'legacy',\n 'replication_manager',\n ),\n result,\n },\n );\n }\n }\n\n /*\n Called when the backup has been continuously stale for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n * backup, so the process is shut down by rejecting the {@link run()}\n * promise. The exit is non-zero and logged at ERROR for alerting; on\n * restart the monitor re-verifies, so a still-wedged backup keeps the\n * process down until the destination is fixed.\n /\n #shutDownOnWedgedBackup(\n staleForMs: number,\n claimedTime: Date,\n lastUpload: Date,\n ) {\n this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n const err = new UnrecoverableError(\n `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n `at ${claimedTime.toISOString()}, but the last object actually ` +\n `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n `not advanced for ${staleForMs} ms (grace period ` +\n `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n `the backup; investigate the backup destination.`,\n );\n this.#lc.error?.(err.message);\n this.#stream.fail(err);\n }\n\n /\n On a fresh stack, fails loudly if no restorable backup has appeared within\n * a generous, replica-size-scaled grace period (see\n * {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the \"give up\" decision\n * onto the replication-manager — the producer responsible for creating the\n * backup — instead of leaving view-syncers to wait (and the platform to keep\n * restarting them) while the real fault is here. Does nothing once a\n * restorable backup has been confirmed.\n /\n async #checkInitialBackupDeadline() {\n if (this.#lastVerifiedUploadTime !== null \|\| this.#runStartTime === null) {\n return;\n }\n const elapsed = Date.now() - this.#runStartTime;\n if (elapsed < this.#initialBackupGraceMs()) {\n return;\n }\n // The deadline has elapsed without a confirmed backup. Do one definitive\n // check against the destination before giving up, in case a backup landed\n // but was never exercised by a reservation or a cleanup verification.\n if (await this.#confirmRestorableBackup()) {\n return;\n }\n this.#shutDownOnMissingInitialBackup(elapsed);\n }\n\n /\n The grace period for the first restorable backup, scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance. Falls back to\n * the minimum (one unit) if the replica file size cannot be determined.\n /\n #initialBackupGraceMs(): number {\n let bytes = 0;\n try {\n bytes = statSync(this.#replicaFile).size;\n } catch {\n // Replica file not present/readable yet; fall back to the minimum grace.\n }\n const units = Math.max(\n 1,\n Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES),\n );\n return units INITIAL_BACKUP_GRACE_MS_PER_UNIT;\n }\n\n /*\n Called when no restorable backup has appeared within\n * {@link #initialBackupGraceMs}. Continuing to run is pointless — no\n * view-syncer can restore — and the fault is the backup pipeline, so the\n * process exits non-zero with a logged error for alerting, mirroring\n * {@link #shutDownOnWedgedBackup}.\n /\n #shutDownOnMissingInitialBackup(elapsedMs: number) {\n const err = new UnrecoverableError(\n `no restorable backup has appeared at ${this.#backupURL} within ` +\n `${elapsedMs} ms of startup (grace period ` +\n `${this.#initialBackupGraceMs()} ms). The initial backup pipeline ` +\n `appears to be broken; shutting down so that the replication-manager ` +\n `is flagged as the cause rather than the view-syncers waiting on it. ` +\n `Investigate litestream replication to ${this.#backupURL}.`,\n );\n this.#lc.error?.(err.message);\n this.#stream.fail(err);\n }\n\n /\n Returns the newest watermark whose backup time is at or before `cutoff`\n * (epoch ms), or `''` if there is none.\n /\n #maxBackupBefore(\n cutoff: number,\n ): {watermark: string; backupTime: Date} \| null {\n let max = '';\n let maxBackupTime: Date \| undefined;\n for (const [watermark, backupTime] of this.#watermarks) {\n if (backupTime.getTime() <= cutoff && watermark > max) {\n max = watermark;\n maxBackupTime = backupTime;\n }\n }\n if (!maxBackupTime) {\n return null;\n }\n return {watermark: max, backupTime: maxBackupTime};\n }\n\n /\n Returns `true` if the actual backup state, as last verified against the\n * backup destination, confirms that data claimed to be backed up at\n * `claimedTime` is durable (i.e. an object was actually uploaded at or\n * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n * clock skew).\n */\n #confirmedDurable(claimedTime: Date): boolean {\n const lastUpload = this.#lastVerifiedUploadTime;\n return (\n lastUpload !== null &&\n claimedTime.getTime() <=\n lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n );\n }\n}\n"],"mappings":";;;;;;;;;;AAgBA,IAAa,oBAAoB;;;;;AAMjC,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B,KAAK;AAE7C,IAAM,MAAM,QAAQ;;;;;;;;;;;;;;;AAgBpB,IAAa,mCAAmC,KAAK;AACrD,IAAM,kCAAkC,KAAK;;;;;;;;;;;;;;;;;AAmC7C,IAAa,8BAAb,MAAyC;CACvC;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,+BAA+B;CAElE;CACA,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAED,iBAAyB;CACzB,oBAAiC;CAMjC,0BAAuC;CAKvC,oBAAmC;CAGnC,gBAA+B;CAC/B;CAEA,YACE,IACA,aACA,WACA,iBACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,+BAA+B;EACtE,KAAKC,eAAe;EACpB,KAAKC,aAAa;EAClB,KAAKC,mBAAmB;EACxB,KAAKI,qBAAqB;EAC1B,KAAKF,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAKI,kBAAkB;GACrC,KAAKL,OAAO,KAAK,KAAKJ,GAAG;EAC3B,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKU,kBAAkB,MAAM,wBAAwB;EAC5D,KAAKA,gBAAgB,KAAK,IAAI;EAC9B,KAAKV,IAAI,OAAO,yBAAyB,KAAKG,kBAAkB;EAChE,KAAKM,qBAAqB,YACxB,KAAK,iCACL,iBACF;EACA,OAAO,KAAKJ;CACd;;;;;;;;;CAUA,MAAMM,2BAA6C;EACjD,IAAI;GACF,KAAKC,0BAA0B,MAAM,KAAKC,wBAAwB;GAClE,OAAO;EACT,SAAS,GAAG;GACV,KAAKb,IAAI,OAAO,gCAAgC,KAAKE,cAAc,CAAC;GACpE,OAAO;EACT;CACF;CAGA,kCAA2C,YAAY;EACrD,IAAI;GACF,MAAM,KAAKY,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKd,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAKY,2BAA2B;EACxC,SAAS,GAAG;GACV,KAAKf,IAAI,OAAO,4BAA4B,CAAC;EAC/C;EACA,IAAI;GACF,MAAM,KAAKgB,4BAA4B;EACzC,SAAS,GAAG;GACV,KAAKhB,IAAI,OAAO,0CAA0C,CAAC;EAC7D;CACF;CAEA,OAAOiB,mBAIJ;EACD,MAAM,kBAAkB,KAAKd;EAC7B,MAAM,SAAS,KAAKC,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAKJ,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAKH,IAAI,OACP,8BAA8B,KAAKG,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GACrD,IAAI,WACF,IAAI;IACF,gBAAgB,SAAS;IACzB,MAAM;KAAC;KAAW;KAAM;IAAI;GAC9B,SAAS,GAAG;IACV,KAAKH,IAAI,OACP,8BAA8B,aAC9B,EAAC,OAAM,GACP,CACF;GACF;EAEJ;CAGN;CAEA,MAAMc,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAKG,iBAAiB,GAChE,IAAI,YAAY,KAAKC,kBAAkB,CAAC,KAAKZ,YAAY,IAAI,SAAS,GAAG;GACvE,KAAKN,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAKM,YAAY,IAAI,WAAW,IAAI;GACpC,KAAKa,oBAAoB;EAC3B;EAEF,OAAO,KAAKA;CACd;CAEA,MAAMJ,6BAA6B;EACjC,MAAM,YAAY,KAAKK,iBAAiB,KAAK,IAAI,CAAC;EAClD,IAAI,CAAC,WACH;EAEF,MAAM,EAAC,WAAW,cAAc,YAAY,gBAAe;EAU3D,IAAI,CAAC,KAAKC,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAKT,0BAA0B,MAAM,KAAKC,wBAAwB;EACpE,SAAS,GAAG;GACV,KAAKL,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAKR,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAKY,uBAAuB;EACpD,MAAM,iBAAiB,KAAKQ,iBAC1B,WAAW,QAAQ,IAAI,4BACzB;EACA,IAAI,CAAC,gBAAgB;GACnB,KAAKZ,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAKc,sBAAsB,MAC7B,KAAKA,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAKA;GAC9B,KAAKtB,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAKE,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAKqB,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAKD,oBAAoB;EACzB,MAAM,EAAC,WAAW,mBAAmB,eAAc;EACnD,KAAK,MAAM,aAAa,KAAKhB,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKD,QAAQ,KAAK;GAChB,WAAW;GACX,cAAc,WAAW,QAAQ;EACnC,CAAC;EACD,KAAKa,iBAAiB;CACxB;CAEA,MAAML,0BAAyC;EAC7C,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAKN,mBAAmB;GAC1C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,EAAE,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAKL,YACL,UACA,qBACF;IACA;GACF,CACF;EACF;CACF;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAKM,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAKN,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;;;;;;CAWA,MAAMW,8BAA8B;EAClC,IAAI,KAAKJ,4BAA4B,QAAQ,KAAKF,kBAAkB,MAClE;EAEF,MAAM,UAAU,KAAK,IAAI,IAAI,KAAKA;EAClC,IAAI,UAAU,KAAKc,sBAAsB,GACvC;EAKF,IAAI,MAAM,KAAKb,yBAAyB,GACtC;EAEF,KAAKc,gCAAgC,OAAO;CAC9C;;;;;;CAOA,wBAAgC;EAC9B,IAAI,QAAQ;EACZ,IAAI;GACF,QAAQ,SAAS,KAAKxB,YAAY,EAAE;EACtC,QAAQ,CAER;EAKA,OAJc,KAAK,IACjB,GACA,KAAK,KAAK,QAAQ,+BAA+B,CAE5C,IAAQ;CACjB;;;;;;;;CASA,gCAAgC,WAAmB;EACjD,MAAM,MAAM,IAAI,mBACd,wCAAwC,KAAKC,WAAW,UACnD,UAAU,+BACV,KAAKsB,sBAAsB,EAAE,kNAGS,KAAKtB,WAAW,EAC7D;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;CAMA,iBACE,QAC8C;EAC9C,IAAI,MAAM;EACV,IAAI;EACJ,KAAK,MAAM,CAAC,WAAW,eAAe,KAAKC,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAAK;GACrD,MAAM;GACN,gBAAgB;EAClB;EAEF,IAAI,CAAC,eACH,OAAO;EAET,OAAO;GAAC,WAAW;GAAK,YAAY;EAAa;CACnD;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAKM;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;AACF"}
1	+ {"version":3,"file":"litestream3-prometheus-poller.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n litestreamBackupVerificationDuration,\n litestreamMonitorMetricAttrs,\n} from '../litestream/metrics.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nexport const CHECK_INTERVAL_MS = 10_000;\n\n/*\n Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n /\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/\n How long the actual backup state may remain continuously behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n \n A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes some such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n \n The change-log is conservatively not purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n /\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 60_000; // 15 minutes\n\nconst GiB = 1024 3;\n\n/\n * How long the BackupMonitor waits for the first restorable backup to appear\n * before concluding that the backup pipeline is broken and shutting the process\n * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).\n \n On a fresh stack the first backup is not restorable until the initial\n * Postgres->replica sync completes and litestream finishes uploading the\n * initial snapshot — a multipart upload that is not listable in the destination\n * until it is committed. The producer cannot observe that sub-snapshot progress,\n * so this is necessarily a generous timeout. It is scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance (~1 hour per\n * 50GB) so that it never fires before the platform would restart a view-syncer\n * that is waiting on the backup.\n /\nexport const INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 60_000; // 1 hour ...\nconst INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB; // ... per 50 GiB\n\n/*\n How often the BackupMonitor re-checks whether the first restorable backup has\n * appeared while a view-syncer holds a snapshot reservation open during a cold\n * start.\n /\nexport const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10_000;\n\n/\n Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n \n See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n /\nexport type BackupStateVerifier = () => Promise<Date>;\n\n/\n The Litestream3PrometheusPoller polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and emits verified watermarks to a stream of {@link BackedUpWatermark}s.\n \n See: https://github.com/rocicorp/litestream/pull/3\n \n Because the watermarks reported by litestream metrics reflect what\n * litestream believes has been backed up (which has been observed to\n * diverge from reality when uploads silently fail), the cleanup watermark is\n * only published after verifying it against the actual backup state in the\n * replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n /\nexport class Litestream3PrometheusPoller {\n readonly #lc: LogContext;\n readonly #replicaFile: string;\n readonly #backupURL: string;\n readonly #metricsEndpoint: string;\n readonly #state = new RunningState('litestream3-prometheus-poller');\n\n readonly #stream: Subscription<BackedUpWatermark>;\n readonly #watermarks = new Map<string, Date>();\n\n readonly #verifyBackupState: BackupStateVerifier;\n readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n description:\n 'Number of change-log purges blocked because the actual backup state ' +\n '(as listed from the replica destination) could not be verified, or ' +\n 'is older than the backup progress claimed by litestream metrics. ' +\n 'A steadily increasing value indicates a wedged or failing backup. ' +\n 'The \"backup-wedged\" reason is emitted once just before the process ' +\n 'shuts itself down due to a persistently stale backup.',\n });\n\n #lastWatermark: string = '';\n #latestBackupTime: Date \| null = null;\n // The time of the most recent object actually verified in the backup\n // destination, or `null` if none has ever been verified. Only assigned from a\n // successful verifyBackupState() and never cleared, so a non-null value also\n // means a restorable backup has been confirmed to exist (which is what gates\n // the snapshot `status` signal and satisfies the initial-backup deadline).\n #lastVerifiedUploadTime: Date \| null = null;\n // Epoch ms at which the actual backup state was first observed to be\n // continuously behind the watermark litestream claims to have backed up,\n // or `null` while the backup is keeping up. Reset whenever a purge is\n // confirmed against a real upload.\n #backupStaleSince: number \| null = null;\n // Epoch ms at which run() started, used to bound how long to wait for the\n // first restorable backup to appear. `null` until run() is called.\n #runStartTime: number \| null = null;\n #checkMetricsTimer: NodeJS.Timeout \| undefined;\n\n constructor(\n lc: LogContext,\n replicaFile: string,\n backupURL: string,\n metricsEndpoint: string,\n verifyBackupState: BackupStateVerifier,\n ) {\n this.#lc = lc.withContext('component', 'litestream3-prometheus-poller');\n this.#replicaFile = replicaFile;\n this.#backupURL = backupURL;\n this.#metricsEndpoint = metricsEndpoint;\n this.#verifyBackupState = verifyBackupState;\n this.#stream = Subscription.create<BackedUpWatermark>({\n cleanup: () => {\n clearInterval(this.#checkMetricsTimer);\n this.#state.stop(this.#lc);\n },\n });\n }\n\n /\n Starts polling for watermarks and pushes new ones to returned stream.\n * Must only be called once.\n /\n start(): Source<BackedUpWatermark> {\n assert(this.#runStartTime === null, `Already called start()`);\n this.#runStartTime = Date.now();\n this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint}`);\n this.#checkMetricsTimer = setInterval(\n this.checkVerifyAndPublishWatermarks,\n CHECK_INTERVAL_MS,\n );\n return this.#stream;\n }\n\n /\n Verifies whether a restorable backup actually exists in the destination\n * (as opposed to what litestream metrics merely claim). On success the\n * verified upload time is cached — seeding the cleanup-verification fast path\n * ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.\n * Returns false if no backup is listable yet (e.g. the initial snapshot is\n * still uploading).\n /\n async #confirmRestorableBackup(): Promise<boolean> {\n try {\n this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n return true;\n } catch (e) {\n this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);\n return false;\n }\n }\n\n // Exported for testing\n readonly checkVerifyAndPublishWatermarks = async () => {\n try {\n await this.#checkWatermarks();\n } catch (e) {\n this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n }\n try {\n await this.#publishVerifiedWatermarks();\n } catch (e) {\n this.#lc.warn?.(`error scheduling cleanup`, e);\n }\n try {\n await this.#checkInitialBackupDeadline();\n } catch (e) {\n this.#lc.warn?.(`error checking initial backup deadline`, e);\n }\n };\n\n async #fetchWatermarks(): AsyncGenerator<{\n watermark: string;\n time: Date;\n name?: string \| undefined;\n }> {\n const metricsEndpoint = this.#metricsEndpoint;\n const signal = this.#state.signal;\n let resp;\n try {\n resp = await fetch(metricsEndpoint, {signal});\n } catch (e) {\n if (signal.aborted) {\n // not an error.\n return;\n }\n // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n // log them and skip the watermark check until the next interval.\n this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n return;\n }\n if (!resp.ok) {\n this.#lc.warn?.(\n `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n );\n return;\n }\n\n const families = parsePrometheusTextFormat(await resp.text());\n for (const family of families) {\n if (\n family.type === 'GAUGE' &&\n family.name === 'litestream_replica_progress'\n ) {\n for (const metric of family.metrics) {\n const watermark = metric.labels?.watermark;\n const name = metric.labels?.name;\n const time = new Date(parseFloat(metric.value) * 1000);\n if (watermark) {\n try {\n versionFromLexi(watermark); // Sanity check: Ensure that it is valid.\n yield {watermark, time, name};\n } catch (e) {\n this.#lc.warn?.(\n `ignoring invalid watermark ${watermark}`,\n {metric},\n e,\n );\n }\n }\n }\n }\n }\n }\n\n async #checkWatermarks() {\n for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n this.#lc.info?.(\n `replicated watermark=${watermark} to ${name}` +\n ` at ${time.toISOString()}.`,\n );\n this.#watermarks.set(watermark, time);\n this.#latestBackupTime = time;\n }\n }\n return this.#latestBackupTime;\n }\n\n async #publishVerifiedWatermarks() {\n const maxBackup = this.#maxBackupBefore(Date.now());\n if (!maxBackup) {\n return;\n }\n const {watermark: maxWatermark, backupTime: claimedTime} = maxBackup;\n // Purge guard: the watermarks (and their backup times) come from\n // litestream metrics, which are exported when litestream believes\n // an upload succeeded, and have been observed to advance even when\n // nothing was actually written to the backup destination. Purging the\n // change-log based on a falsely advancing watermark permanently breaks\n // the ability to restore + catch up. Before advancing the cleanup\n // watermark, verify it against the actual backup state: a claimed\n // backup time is only trusted if an object was actually uploaded to\n // the replica destination at (or after) that time, modulo clock skew.\n if (!this.#confirmedDurable(claimedTime)) {\n try {\n this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n } catch (e) {\n this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n // Skipping the purge is safe: the change-log just grows.\n this.#lc.warn?.(\n `unable to verify backup state. skipping change-log cleanup ` +\n `up to watermark ${maxWatermark} ` +\n `(claimed backup time ${claimedTime.toISOString()})`,\n e,\n );\n return;\n }\n }\n // Watermarks whose backup time isn't yet confirmed durable remain in the\n // map and are re-evaluated at the next check.\n const lastUpload = must(this.#lastVerifiedUploadTime);\n const verifiedBackup = this.#maxBackupBefore(\n lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n );\n if (!verifiedBackup) {\n this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n const now = Date.now();\n if (this.#backupStaleSince === null) {\n this.#backupStaleSince = now;\n }\n const staleForMs = now - this.#backupStaleSince;\n this.#lc.warn?.(\n `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n `litestream claims it was backed up at ` +\n `${claimedTime.toISOString()}, but the last object actually ` +\n `uploaded to ${this.#backupURL} was at ` +\n `${lastUpload.toISOString()}. ` +\n `The backup has been stale for ${staleForMs} ms.`,\n );\n if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n }\n return;\n }\n // The cleanup watermark advanced past a real upload, so the backup is\n // keeping up: clear the staleness clock.\n this.#backupStaleSince = null;\n const {watermark: verifiedWatermark, backupTime} = verifiedBackup;\n for (const watermark of this.#watermarks.keys()) {\n if (watermark <= verifiedWatermark) {\n this.#watermarks.delete(watermark);\n }\n }\n this.#stream.push({\n watermark: verifiedWatermark,\n backupTimeMs: backupTime.getTime(),\n });\n this.#lastWatermark = verifiedWatermark;\n }\n\n async #verifyBackupStateTimed(): Promise<Date> {\n const start = performance.now();\n let result: 'success' \| 'error' = 'error';\n try {\n const ret = await this.#verifyBackupState();\n result = 'success';\n return ret;\n } finally {\n litestreamBackupVerificationDuration().recordMs(\n performance.now() - start,\n {\n ...litestreamMonitorMetricAttrs(\n this.#backupURL,\n 'legacy',\n 'replication_manager',\n ),\n result,\n },\n );\n }\n }\n\n /*\n Called when the backup has been continuously stale for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n * backup, so the process is shut down by rejecting the {@link run()}\n * promise. The exit is non-zero and logged at ERROR for alerting; on\n * restart the monitor re-verifies, so a still-wedged backup keeps the\n * process down until the destination is fixed.\n /\n #shutDownOnWedgedBackup(\n staleForMs: number,\n claimedTime: Date,\n lastUpload: Date,\n ) {\n this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n const err = new UnrecoverableError(\n `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n `at ${claimedTime.toISOString()}, but the last object actually ` +\n `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n `not advanced for ${staleForMs} ms (grace period ` +\n `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n `the backup; investigate the backup destination.`,\n );\n this.#lc.error?.(err.message);\n this.#stream.fail(err);\n }\n\n /\n On a fresh stack, fails loudly if no restorable backup has appeared within\n * a generous, replica-size-scaled grace period (see\n * {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the \"give up\" decision\n * onto the replication-manager — the producer responsible for creating the\n * backup — instead of leaving view-syncers to wait (and the platform to keep\n * restarting them) while the real fault is here. Does nothing once a\n * restorable backup has been confirmed.\n /\n async #checkInitialBackupDeadline() {\n if (this.#lastVerifiedUploadTime !== null \|\| this.#runStartTime === null) {\n return;\n }\n const elapsed = Date.now() - this.#runStartTime;\n if (elapsed < this.#initialBackupGraceMs()) {\n return;\n }\n // The deadline has elapsed without a confirmed backup. Do one definitive\n // check against the destination before giving up, in case a backup landed\n // but was never exercised by a reservation or a cleanup verification.\n if (await this.#confirmRestorableBackup()) {\n return;\n }\n this.#shutDownOnMissingInitialBackup(elapsed);\n }\n\n /\n The grace period for the first restorable backup, scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance. Falls back to\n * the minimum (one unit) if the replica file size cannot be determined.\n /\n #initialBackupGraceMs(): number {\n let bytes = 0;\n try {\n bytes = statSync(this.#replicaFile).size;\n } catch {\n // Replica file not present/readable yet; fall back to the minimum grace.\n }\n const units = Math.max(\n 1,\n Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES),\n );\n return units INITIAL_BACKUP_GRACE_MS_PER_UNIT;\n }\n\n /*\n Called when no restorable backup has appeared within\n * {@link #initialBackupGraceMs}. Continuing to run is pointless — no\n * view-syncer can restore — and the fault is the backup pipeline, so the\n * process exits non-zero with a logged error for alerting, mirroring\n * {@link #shutDownOnWedgedBackup}.\n /\n #shutDownOnMissingInitialBackup(elapsedMs: number) {\n const err = new UnrecoverableError(\n `no restorable backup has appeared at ${this.#backupURL} within ` +\n `${elapsedMs} ms of startup (grace period ` +\n `${this.#initialBackupGraceMs()} ms). The initial backup pipeline ` +\n `appears to be broken; shutting down so that the replication-manager ` +\n `is flagged as the cause rather than the view-syncers waiting on it. ` +\n `Investigate litestream replication to ${this.#backupURL}.`,\n );\n this.#lc.error?.(err.message);\n this.#stream.fail(err);\n }\n\n /\n Returns the newest watermark whose backup time is at or before `cutoff`\n * (epoch ms), or `''` if there is none.\n /\n #maxBackupBefore(\n cutoff: number,\n ): {watermark: string; backupTime: Date} \| null {\n let max = '';\n let maxBackupTime: Date \| undefined;\n for (const [watermark, backupTime] of this.#watermarks) {\n if (backupTime.getTime() <= cutoff && watermark > max) {\n max = watermark;\n maxBackupTime = backupTime;\n }\n }\n if (!maxBackupTime) {\n return null;\n }\n return {watermark: max, backupTime: maxBackupTime};\n }\n\n /\n Returns `true` if the actual backup state, as last verified against the\n * backup destination, confirms that data claimed to be backed up at\n * `claimedTime` is durable (i.e. an object was actually uploaded at or\n * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n * clock skew).\n */\n #confirmedDurable(claimedTime: Date): boolean {\n const lastUpload = this.#lastVerifiedUploadTime;\n return (\n lastUpload !== null &&\n claimedTime.getTime() <=\n lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n );\n }\n}\n"],"mappings":";;;;;;;;;;AAgBA,IAAa,oBAAoB;;;;;AAMjC,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B;AAExC,IAAM,MAAM,QAAQ;;;;;;;;;;;;;;;AAgBpB,IAAa,mCAAmC;AAChD,IAAM,kCAAkC,KAAK;;;;;;;;;;;;;;;;;AAmC7C,IAAa,8BAAb,MAAyC;CACvC;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,+BAA+B;CAElE;CACA,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAED,iBAAyB;CACzB,oBAAiC;CAMjC,0BAAuC;CAKvC,oBAAmC;CAGnC,gBAA+B;CAC/B;CAEA,YACE,IACA,aACA,WACA,iBACA,mBACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,+BAA+B;EACtE,KAAK,eAAe;EACpB,KAAK,aAAa;EAClB,KAAK,mBAAmB;EACxB,KAAK,qBAAqB;EAC1B,KAAK,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAK,kBAAkB;GACrC,KAAK,OAAO,KAAK,KAAK,GAAG;EAC3B,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAK,kBAAkB,MAAM,wBAAwB;EAC5D,KAAK,gBAAgB,KAAK,IAAI;EAC9B,KAAK,IAAI,OAAO,yBAAyB,KAAK,kBAAkB;EAChE,KAAK,qBAAqB,YACxB,KAAK,iCACL,iBACF;EACA,OAAO,KAAK;CACd;;;;;;;;;CAUA,MAAM,2BAA6C;EACjD,IAAI;GACF,KAAK,0BAA0B,MAAM,KAAK,wBAAwB;GAClE,OAAO;EACT,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,gCAAgC,KAAK,cAAc,CAAC;GACpE,OAAO;EACT;CACF;CAGA,kCAA2C,YAAY;EACrD,IAAI;GACF,MAAM,KAAK,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,8BAA8B,KAAK,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAK,2BAA2B;EACxC,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,4BAA4B,CAAC;EAC/C;EACA,IAAI;GACF,MAAM,KAAK,4BAA4B;EACzC,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,0CAA0C,CAAC;EAC7D;CACF;CAEA,OAAO,mBAIJ;EACD,MAAM,kBAAkB,KAAK;EAC7B,MAAM,SAAS,KAAK,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAK,IAAI,OAAO,8BAA8B,KAAK,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAK,IAAI,OACP,8BAA8B,KAAK,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GACrD,IAAI,WACF,IAAI;IACF,gBAAgB,SAAS;IACzB,MAAM;KAAC;KAAW;KAAM;IAAI;GAC9B,SAAS,GAAG;IACV,KAAK,IAAI,OACP,8BAA8B,aAC9B,EAAC,OAAM,GACP,CACF;GACF;EAEJ;CAGN;CAEA,MAAM,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAK,iBAAiB,GAChE,IAAI,YAAY,KAAK,kBAAkB,CAAC,KAAK,YAAY,IAAI,SAAS,GAAG;GACvE,KAAK,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAK,YAAY,IAAI,WAAW,IAAI;GACpC,KAAK,oBAAoB;EAC3B;EAEF,OAAO,KAAK;CACd;CAEA,MAAM,6BAA6B;EACjC,MAAM,YAAY,KAAK,iBAAiB,KAAK,IAAI,CAAC;EAClD,IAAI,CAAC,WACH;EAEF,MAAM,EAAC,WAAW,cAAc,YAAY,gBAAe;EAU3D,IAAI,CAAC,KAAK,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAK,0BAA0B,MAAM,KAAK,wBAAwB;EACpE,SAAS,GAAG;GACV,KAAK,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAK,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAK,uBAAuB;EACpD,MAAM,iBAAiB,KAAK,iBAC1B,WAAW,QAAQ,IAAI,4BACzB;EACA,IAAI,CAAC,gBAAgB;GACnB,KAAK,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAK,sBAAsB,MAC7B,KAAK,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAK;GAC9B,KAAK,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAK,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAK,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAK,oBAAoB;EACzB,MAAM,EAAC,WAAW,mBAAmB,eAAc;EACnD,KAAK,MAAM,aAAa,KAAK,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAK,YAAY,OAAO,SAAS;EAGrC,KAAK,QAAQ,KAAK;GAChB,WAAW;GACX,cAAc,WAAW,QAAQ;EACnC,CAAC;EACD,KAAK,iBAAiB;CACxB;CAEA,MAAM,0BAAyC;EAC7C,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAK,mBAAmB;GAC1C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,CAAC,CAAC,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAK,YACL,UACA,qBACF;IACA;GACF,CACF;EACF;CACF;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAK,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAK,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAK,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAK,QAAQ,KAAK,GAAG;CACvB;;;;;;;;;;CAWA,MAAM,8BAA8B;EAClC,IAAI,KAAK,4BAA4B,QAAQ,KAAK,kBAAkB,MAClE;EAEF,MAAM,UAAU,KAAK,IAAI,IAAI,KAAK;EAClC,IAAI,UAAU,KAAK,sBAAsB,GACvC;EAKF,IAAI,MAAM,KAAK,yBAAyB,GACtC;EAEF,KAAK,gCAAgC,OAAO;CAC9C;;;;;;CAOA,wBAAgC;EAC9B,IAAI,QAAQ;EACZ,IAAI;GACF,QAAQ,SAAS,KAAK,YAAY,CAAC,CAAC;EACtC,QAAQ,CAER;EAKA,OAJc,KAAK,IACjB,GACA,KAAK,KAAK,QAAQ,+BAA+B,CAE5C,IAAQ;CACjB;;;;;;;;CASA,gCAAgC,WAAmB;EACjD,MAAM,MAAM,IAAI,mBACd,wCAAwC,KAAK,WAAW,UACnD,UAAU,+BACV,KAAK,sBAAsB,EAAE,kNAGS,KAAK,WAAW,EAC7D;EACA,KAAK,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAK,QAAQ,KAAK,GAAG;CACvB;;;;;CAMA,iBACE,QAC8C;EAC9C,IAAI,MAAM;EACV,IAAI;EACJ,KAAK,MAAM,CAAC,WAAW,eAAe,KAAK,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAAK;GACrD,MAAM;GACN,gBAAgB;EAClB;EAEF,IAAI,CAAC,eACH,OAAO;EAET,OAAO;GAAC,WAAW;GAAK,YAAY;EAAa;CACnD;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAK;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;AACF"}

package/out/zero-cache/src/services/change-streamer/replica-poller.js CHANGED Viewed

@@ -5,7 +5,7 @@ import { Subscription } from "../../types/subscription.js";
 import { assert } from "console";
 import { clearInterval } from "timers";
 //#region ../zero-cache/src/services/change-streamer/replica-poller.ts
-var CHECK_INTERVAL_MS = 10 * 1e3;
+var CHECK_INTERVAL_MS = 1e4;
 /**
 * The single-node equivalent of a backup monitor polls the replica file every
 * 30 seconds and schedules cleanup when the watermark

package/out/zero-cache/src/services/change-streamer/replica-poller.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"replica-poller.js","names":[~~"#lc","#replicaFile","#stream","#pollReplicaTimer","#pollReplica","#lastWatermark"~~],"sources":["../../../../../../zero-cache/src/services/change-streamer/replica-poller.ts"],"sourcesContent":["import {assert} from 'console';\nimport {clearInterval} from 'timers';\nimport type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nconst CHECK_INTERVAL_MS = 10 * 1000;\n\n/*\n The single-node equivalent of a backup monitor polls the replica file every\n * 30 seconds and schedules cleanup when the watermark\n * (i.e. stateVersion) moves forward.\n */\nexport class ReplicaPoller {\n readonly id = 'replica-poller';\n readonly #lc: LogContext;\n readonly #replicaFile: string;\n readonly #stream: Subscription<BackedUpWatermark>;\n #pollReplicaTimer: NodeJS.Timeout \| undefined;\n\n #lastWatermark: string = '';\n\n constructor(lc: LogContext, replicaFile: string) {\n this.#lc = lc.withContext('component', this.id);\n this.#replicaFile = replicaFile;\n this.#stream = Subscription.create<BackedUpWatermark>({\n cleanup: () => clearInterval(this.#pollReplicaTimer),\n });\n }\n\n start(): Source<BackedUpWatermark> {\n assert(this.#pollReplicaTimer === undefined, `Already called start()`);\n this.#lc.info?.(`starting replica monitor`);\n // Perform the first poll immediately to avoid unnecessarily waiting\n // during single-node development.\n this.#pollReplica();\n this.#pollReplicaTimer = setInterval(this.#pollReplica, CHECK_INTERVAL_MS);\n return this.#stream;\n }\n\n readonly #pollReplica = () => {\n const db = new Database(this.#lc, this.#replicaFile);\n try {\n const {stateVersion} = getReplicationState(new StatementRunner(db));\n if (stateVersion !== this.#lastWatermark) {\n this.#lastWatermark = stateVersion;\n this.#lc.debug?.(`replicated up to watermark ${stateVersion}`);\n this.#stream.push({watermark: stateVersion, backupTimeMs: Date.now()});\n }\n } catch (e) {\n this.#lc.error?.(`Unable to read watermark from replica`, e);\n } finally {\n db.close();\n }\n };\n}\n"],"mappings":";;;;;;;AAUA,IAAM,oBAAoB~~,KAAK~~;;;;;;~~AAO/B~~,IAAa,gBAAb,MAA2B;CACzB,KAAc;CACd;CACA;CACA;CACA;CAEA,iBAAyB;CAEzB,YAAY,IAAgB,aAAqB;EAC/C,~~KAAKA~~,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,~~KAAKC~~,eAAe;EACpB,~~KAAKC~~,UAAU,aAAa,OAA0B,EACpD,eAAe,cAAc,~~KAAKC~~,iBAAiB,EACrD,CAAC;CACH;CAEA,QAAmC;EACjC,OAAO,~~KAAKA~~,sBAAsB,KAAA,GAAW,wBAAwB;EACrE,~~KAAKH~~,IAAI,OAAO,0BAA0B;EAG1C,~~KAAKI~~,aAAa;EAClB,~~KAAKD~~,oBAAoB,YAAY,~~KAAKC~~,cAAc,iBAAiB;EACzE,OAAO,~~KAAKF~~;CACd;CAEA,qBAA8B;EAC5B,MAAM,KAAK,IAAI,SAAS,~~KAAKF~~,KAAK,~~KAAKC~~,YAAY;EACnD,IAAI;GACF,MAAM,EAAC,iBAAgB,oBAAoB,IAAI,gBAAgB,EAAE,CAAC;GAClE,IAAI,iBAAiB,~~KAAKI~~,gBAAgB;IACxC,~~KAAKA~~,iBAAiB;IACtB,~~KAAKL~~,IAAI,QAAQ,8BAA8B,cAAc;IAC7D,~~KAAKE~~,QAAQ,KAAK;KAAC,WAAW;KAAc,cAAc,KAAK,IAAI;IAAC,CAAC;GACvE;EACF,SAAS,GAAG;GACV,~~KAAKF~~,IAAI,QAAQ,yCAAyC,CAAC;EAC7D,UAAU;GACR,GAAG,MAAM;EACX;CACF;AACF"}
1	+ {"version":3,"file":"replica-poller.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/replica-poller.ts"],"sourcesContent":["import {assert} from 'console';\nimport {clearInterval} from 'timers';\nimport type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nconst CHECK_INTERVAL_MS = 10 * 1000;\n\n/*\n The single-node equivalent of a backup monitor polls the replica file every\n * 30 seconds and schedules cleanup when the watermark\n * (i.e. stateVersion) moves forward.\n */\nexport class ReplicaPoller {\n readonly id = 'replica-poller';\n readonly #lc: LogContext;\n readonly #replicaFile: string;\n readonly #stream: Subscription<BackedUpWatermark>;\n #pollReplicaTimer: NodeJS.Timeout \| undefined;\n\n #lastWatermark: string = '';\n\n constructor(lc: LogContext, replicaFile: string) {\n this.#lc = lc.withContext('component', this.id);\n this.#replicaFile = replicaFile;\n this.#stream = Subscription.create<BackedUpWatermark>({\n cleanup: () => clearInterval(this.#pollReplicaTimer),\n });\n }\n\n start(): Source<BackedUpWatermark> {\n assert(this.#pollReplicaTimer === undefined, `Already called start()`);\n this.#lc.info?.(`starting replica monitor`);\n // Perform the first poll immediately to avoid unnecessarily waiting\n // during single-node development.\n this.#pollReplica();\n this.#pollReplicaTimer = setInterval(this.#pollReplica, CHECK_INTERVAL_MS);\n return this.#stream;\n }\n\n readonly #pollReplica = () => {\n const db = new Database(this.#lc, this.#replicaFile);\n try {\n const {stateVersion} = getReplicationState(new StatementRunner(db));\n if (stateVersion !== this.#lastWatermark) {\n this.#lastWatermark = stateVersion;\n this.#lc.debug?.(`replicated up to watermark ${stateVersion}`);\n this.#stream.push({watermark: stateVersion, backupTimeMs: Date.now()});\n }\n } catch (e) {\n this.#lc.error?.(`Unable to read watermark from replica`, e);\n } finally {\n db.close();\n }\n };\n}\n"],"mappings":";;;;;;;AAUA,IAAM,oBAAoB;;;;;;AAO1B,IAAa,gBAAb,MAA2B;CACzB,KAAc;CACd;CACA;CACA;CACA;CAEA,iBAAyB;CAEzB,YAAY,IAAgB,aAAqB;EAC/C,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAK,eAAe;EACpB,KAAK,UAAU,aAAa,OAA0B,EACpD,eAAe,cAAc,KAAK,iBAAiB,EACrD,CAAC;CACH;CAEA,QAAmC;EACjC,OAAO,KAAK,sBAAsB,KAAA,GAAW,wBAAwB;EACrE,KAAK,IAAI,OAAO,0BAA0B;EAG1C,KAAK,aAAa;EAClB,KAAK,oBAAoB,YAAY,KAAK,cAAc,iBAAiB;EACzE,OAAO,KAAK;CACd;CAEA,qBAA8B;EAC5B,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,YAAY;EACnD,IAAI;GACF,MAAM,EAAC,iBAAgB,oBAAoB,IAAI,gBAAgB,EAAE,CAAC;GAClE,IAAI,iBAAiB,KAAK,gBAAgB;IACxC,KAAK,iBAAiB;IACtB,KAAK,IAAI,QAAQ,8BAA8B,cAAc;IAC7D,KAAK,QAAQ,KAAK;KAAC,WAAW;KAAc,cAAc,KAAK,IAAI;IAAC,CAAC;GACvE;EACF,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,yCAAyC,CAAC;EAC7D,UAAU;GACR,GAAG,MAAM;EACX;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/restore-admission.js CHANGED Viewed

@@ -1,4 +1,15 @@
 import { getOrCreateGauge } from "../../observability/metrics.js";
+//#region ../zero-cache/src/services/change-streamer/restore-admission.ts
+/**
+* The default time to wait for a subscriber that was granted permission to
+* restore to reconnect (i.e. to open a replacement reservation) after its
+* held connection closes, or after its replacement reservation closes
+* without subscribing (e.g. to retry a failed restore). This covers the
+* subscriber's process exit and restart (e.g. a container restart, the
+* backoff of which starts at 10 seconds in Kubernetes), but not the restore
+* itself, which is tracked by the liveness of its snapshot reservation.
+*/
+var DEFAULT_PENDING_GRANT_TTL_MS = 9e4;
 /**
 * The default time for which a replacement continues to be counted after its
 * subscriber has caught up, which covers the time for the subscriber to start
@@ -312,6 +323,6 @@ var RestoreAdmission = class {
 	}
 };
 //#endregion
-export { DEFAULT_READINESS_GRACE_PERIOD_MS, RestoreAdmission };
+export { DEFAULT_PENDING_GRANT_TTL_MS, DEFAULT_READINESS_GRACE_PERIOD_MS, RestoreAdmission };
 //# sourceMappingURL=restore-admission.js.map