npm - @rocicorp/zero - Versions diffs - 1.10.0-canary.2 → 1.10.0-canary.20 - Mend

@rocicorp/zero 1.10.0-canary.2 → 1.10.0-canary.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (372) hide show

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#reservations","#replicationStatusPublisher","#sqliteCatchupOptions","#changeLogWriter","#purgeScheduler","#acker","#initializer","#autoReset","#state","#serving","#txCounter","#changeCounter","#transactionForwardDuration","#sqliteCatchup","#closeSQLiteCatchup","#purgeLock","#latestStatus","#latestLagReportCommitTimeMs","#lastForwardedCommitWatermark","#stream","#handleControlMessage","#recordForwardedTransactionBoundary","#selectSQLiteCatchup","#captureRequiredHead","#confirmReservations","#backupWatermark","#requestSQLitePurge","#maybeSchedulePGPurge","#maybeScheduleSQLitePurge","#getChangeLogState","#pgPurgeScheduled","#pgPurgeRunning","#pgPurgedWatermark","#purgePGChangeLog","#sqlitePurgeScheduled","#sqlitePurgeRunning","#sqlitePurgeContinuation","#purgeSQLiteChangeLog","#getCleanupFloor","#loggedBehindWatermark","#currentTransactionCompletion","#transactionForwardStartedAt","#recordTransactionForwardDuration","#openSQLiteCatchup","#declineSQLiteCatchup","#changeLogUnavailableSince"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import {getDefaultHighWaterMark} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {\n getOrCreateCounter,\n getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {min} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n ChangeSource,\n ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n type ChangeStreamControl,\n type ChangeStreamData,\n type Rollback,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {LitestreamVersion} from '../litestream/metrics.ts';\nimport {\n publishReplicationError,\n replicationStatusError,\n type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n DEFAULT_MAX_RETRY_DELAY_MS,\n RunningState,\n UnrecoverableError,\n} from '../running-state.ts';\nimport {\n ChangeLogInitializer,\n replicaInitializationSource,\n} from './change-log-initializer.ts';\nimport {\n type ChangeStreamerService,\n type Status,\n type SubscriberContext,\n type WatermarkedChange,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {\n AutoResetSignal,\n ensureReplicationConfig,\n markResetRequired,\n} from './schema/tables.ts';\nimport {SnapshotReservations} from './snapshot-reservations.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\nimport {\n SQLiteChangeLogCatchup,\n type SQLiteChangeLogCleanupGuard,\n} from './sqlite-change-log-catchup.ts';\nimport {\n SQLiteChangeLogPurgeScheduler,\n type PurgeContinuation,\n type SQLiteChangeLogPurgeSchedulerOptions,\n} from './sqlite-change-log-purge-scheduler.ts';\nimport {SQLiteChangeLogReader} from './sqlite-change-log-reader.ts';\nimport {\n SQLiteChangeLogWriter,\n type SQLiteChangeLogWriterOptions,\n} from './sqlite-change-log-writer.ts';\nimport {\n Storer,\n type PurgeLock,\n type TuningOptions as StorerOptions,\n} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\nimport {UpstreamAcker} from './upstream-acker.ts';\n\nexport type BackupConfig = {\n backupURL: string;\n litestreamVersion: LitestreamVersion;\n};\n\nexport type SQLiteCatchupOptions = {\n /** The change-log database, i.e. `changeLogFileName(replicaFile)`. /\n changeLogFile: string;\n readBatchRows: number;\n barrierTimeoutMs: number;\n /\n Backstop for the change-log writer's ACK, which is what normally releases\n * the barrier. Defaults to the catchup coordinator's interval.\n /\n barrierPollIntervalMs?: number \| undefined;\n /\n Slice 11 supplies the production canary selector.\n \n It does not need to exclude backup subscribers or the change-log writer:\n * eligibility checks reject both before invoking this selector. Serving a\n * writer from SQLite would make it wait on its own ACK.\n /\n shouldUse?: ((ctx: SubscriberContext) => boolean) \| undefined;\n /\n Overrides the purge scheduler's guard, for tests. When absent, the\n * service supplies the scheduler's real writer-serialized guard, or the\n * catchup's no-op default when purging is not configured.\n /\n cleanupGuard?: SQLiteChangeLogCleanupGuard \| undefined;\n /\n How long the change log may be unavailable before declining to serve from\n * it is reported as a warning rather than a debug line. Defaults to\n * {@link DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS}.\n /\n notReadyWarnThresholdMs?: number \| undefined;\n};\n\nexport type TuningOptions = StorerOptions & {\n flowControlConsensusTimeoutProportion: number;\n flowControlSlowSubscriberGracePeriodMs?: number \| undefined;\n sqliteCatchup?: SQLiteCatchupOptions \| undefined;\n /\n Supplied when `sqliteChangeLogMode != off`, i.e. this is the gate on the\n * change-log writer. Absent, nothing writes the log.\n /\n sqliteChangeLogWriter?:\n \| Omit<\n SQLiteChangeLogWriterOptions,\n 'onCommit' \| 'onDisabled' \| 'onRebuilt'\n >\n \| undefined;\n /\n Also supplied when `sqliteChangeLogMode != off`, and deliberately not\n * gated on the read path: in `write` mode no reader ever opens, and this is\n * the configuration the purge scheduler actually ships in. The scheduler\n * runs on the writer's own connection, so with the writer absent (mode\n * `off`) it is never constructed, and a leftover file on disk is left to\n * the replicator's cleanup.\n /\n sqliteChangeLogPurge?: SQLiteChangeLogPurgeSchedulerOptions \| undefined;\n /\n Supplied at `sqliteChangeLogMode >= compare`, and the gate on deriving the\n * stream's initialization parameters from the replica as well as from\n * Postgres. Postgres stays authoritative; the two are compared and the\n * result is reported. Not a new configuration option -- the mode ladder\n * carries it, because a state in which the log's buffer and its cookies were\n * at different rollout stages is what invariant 15 exists to forbid.\n /\n sqliteChangeLogCompare?: {replicaFile: string} \| undefined;\n};\n\n/\n Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n /\nexport async function initializeStreamer(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n changeSource: ChangeSource,\n replicationStatusPublisher: ReplicationStatusPublisher,\n subscriptionState: SubscriptionState,\n backupConfig: BackupConfig \| null,\n purgeLock: PurgeLock \| null,\n autoReset: boolean,\n opts: TuningOptions,\n setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n await ensureReplicationConfig(\n lc,\n changeDB,\n subscriptionState,\n shard,\n autoReset,\n purgeLock ?? undefined,\n setTimeoutFn,\n );\n\n const {replicaVersion} = subscriptionState;\n return new ChangeStreamerImpl(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n changeSource,\n replicationStatusPublisher,\n backupConfig,\n purgeLock,\n autoReset,\n opts,\n setTimeoutFn,\n );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\n// How long the change log may be unavailable before declining to serve from it\n// stops looking like normal startup ordering. Subscriptions can arrive before\n// the stream loop's first reconcile has created and seeded the log, so early\n// declines are expected. Still declining a minute later means the log is not\n// being written at all -- `sqliteChangeLogMode=off`, a writer that has failed\n// soft, or a path mismatch -- which is worth surfacing.\nconst DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS = 60_000;\n\n/\n Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n \n ### Subscriber Catchup\n \n Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n \n \n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * \| Historic changes in storage \| Pending (streamed) tx \| Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * Replication stream\n * > > > > > > > > >\n * ^ ---> required catchup ---> ^\n * Subscriber watermark Subscription begins\n * ```\n \n Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n \n Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n \n * If the streamer is in the middle of a pending Transaction, the\n * Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n * state, new changes are not forwarded to the Subscriber, and catchup\n * is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n * by the Storer, it begins catchup on the Subscriber (with a READONLY\n * snapshot so that it does not block subsequent storage operations).\n * This catchup is thus guaranteed to load the change log entries of\n * that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n * Subscriber from the \"queued\" to the \"active\" set of clients such that\n * the Subscriber begins receiving new changes, starting from the next\n * Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n * is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n * and the Subscriber henceforth forwards changes as they are received.\n \n In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n \n ### Watermarks and ordering\n \n The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n \n https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n \n > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n \n In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n \n A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n \n ```\n * LSN: 1 2 3 4 5 6 7 8 9 10\n * tx1: begin data data data commit\n * tx2: begin data data data commit\n * ```\n \n will arrive as:\n \n ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n \n Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n \n Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n \n * A `commit` message is assigned to a watermark corresponding to its LSN.\n * These are guaranteed to be in commit order by definition.\n \n * `begin` and `data` messages are assigned to the watermark of the\n * preceding `commit` (the previous transaction, or the replication\n * slot's starting LSN) plus 1. This guarantees that they will be sorted\n * after the previously commit transaction even if their LSNs came before it.\n * This is referred to as the `preCommitWatermark`.\n \n * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n * the position of the message within its transaction, with the `begin` message\n * starting at `0`. This guarantees that `begin` and `data` messages will be\n * fetched in the original ChangeSource order during catchup.\n \n `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n \n ### Cleanup\n \n As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n \n More concretely:\n \n * The `initial`, backup-derived watermark is the earliest to which cleanup\n * should ever happen.\n \n * However, it is possible for the replica backup to be ahead of a connected\n * subscriber; and if a network error causes that subscriber to retry from its\n * last watermark, the change streamer must support it.\n \n Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n /\nclass ChangeStreamerImpl implements ChangeStreamerService {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #replicaVersion: string;\n readonly #source: ChangeSource;\n readonly #storer: Storer;\n readonly #forwarder: Forwarder;\n readonly #reservations: SnapshotReservations \| undefined;\n readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n readonly #sqliteCatchupOptions: SQLiteCatchupOptions \| undefined;\n readonly #changeLogWriter: SQLiteChangeLogWriter \| undefined;\n readonly #purgeScheduler: SQLiteChangeLogPurgeScheduler \| undefined;\n readonly #acker: UpstreamAcker;\n readonly #initializer: ChangeLogInitializer;\n\n readonly #autoReset: boolean;\n readonly #state: RunningState;\n\n // Starting the (Postgres) ChangeStream results in killing the previous\n // Postgres subscriber, potentially creating a gap in which the old\n // change-streamer has shut down and the new change-streamer has not yet\n // been recognized as \"healthy\" (and thus does not get any requests).\n //\n // To minimize this gap, delay starting the ChangeStream until the first\n // request from a `serving` replicator, indicating that higher level\n // load-balancing / routing logic has begun routing requests to this task.\n readonly #serving = resolver();\n\n readonly #txCounter = getOrCreateCounter(\n 'replication',\n 'transactions',\n 'Count of replicated transactions',\n );\n readonly #changeCounter = getOrCreateCounter(\n 'replication',\n 'changes',\n 'Count of replicated changes (DML or DDL statements)',\n );\n // The number the SQLite change-log commit lands in. It is labeled by whether\n // the log is being written so that the cost of putting a commit on the\n // forward path is attributable rather than inferred.\n readonly #transactionForwardDuration = getOrCreateLatencyHistogram(\n 'replication',\n 'transaction_forward_duration',\n \"Time from receiving a transaction's `begin` to forwarding its `commit`, \" +\n 'i.e. the change-streamer half of forward-to-subscriber latency.',\n );\n\n #latestStatus: Status;\n #latestLagReportCommitTimeMs = 0;\n #backupWatermark: string \| undefined;\n #pgPurgedWatermark: string = '';\n /\n The floor last logged as held back by a laggard. The level-triggered\n * retry re-evaluates every {@link CLEANUP_DELAY_MS}; logging only when the\n * blocking floor moves keeps a stuck subscriber from emitting the same\n * line indefinitely.\n /\n #loggedBehindWatermark: string \| undefined;\n #sqlitePurgeContinuation: PurgeContinuation \| undefined;\n #purgeLock: PurgeLock \| null;\n // PG and SQLite intentionally own separate level-triggered loops. Neither\n // waits for, advances, or retries the other.\n #pgPurgeScheduled = false;\n #pgPurgeRunning = false;\n #sqlitePurgeScheduled = false;\n #sqlitePurgeRunning = false;\n #stream: ChangeStream \| undefined;\n #sqliteCatchup: SQLiteChangeLogCatchup \| undefined;\n #changeLogUnavailableSince: number \| undefined;\n #lastForwardedCommitWatermark: string \| undefined;\n #transactionForwardStartedAt: number \| undefined;\n #currentTransactionCompletion:\n \| Resolver<ForwardedTransactionCompletion>\n \| undefined;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n replicaVersion: string,\n source: ChangeSource,\n replicationStatusPublisher: ReplicationStatusPublisher,\n backupConfig: BackupConfig \| null,\n initialPurgeLock: PurgeLock \| null,\n autoReset: boolean,\n opts: TuningOptions,\n setTimeoutFn = setTimeout,\n ) {\n this.id = `change-streamer`;\n this.#lc = lc.withContext('component', 'change-streamer');\n this.#shard = shard;\n this.#changeDB = changeDB;\n this.#replicaVersion = replicaVersion;\n this.#source = source;\n this.#storer = new Storer(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n consumed => this.#acker.trackPgChangeLog(consumed[2].watermark),\n err => this.stop(err),\n opts,\n );\n this.#forwarder = new Forwarder(lc, {\n flowControlConsensusTimeoutProportion:\n opts.flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodMs:\n opts.flowControlSlowSubscriberGracePeriodMs,\n });\n this.#reservations = backupConfig\n ? new SnapshotReservations(lc, backupConfig, taskID =>\n this.#purgeScheduler?.resume(taskID),\n )\n : undefined;\n this.#replicationStatusPublisher = replicationStatusPublisher;\n this.#changeLogWriter = opts.sqliteChangeLogWriter\n ? new SQLiteChangeLogWriter(lc, {\n ...opts.sqliteChangeLogWriter,\n onCommit: watermark => {\n this.#sqliteCatchup?.onChangeLogCommit(watermark);\n // The commit closed the log's transaction, i.e. opened a purge\n // window (§3.3).\n this.#purgeScheduler?.onWriterIdle();\n },\n // Fail-soft deletes the file, and the reader is cached here, so it\n // has to go with it: otherwise it serves an unlinked inode while\n // every new open sees nothing.\n onDisabled: () => this.#closeSQLiteCatchup(),\n onRebuilt: () => this.#closeSQLiteCatchup(),\n })\n : undefined;\n // The purge scheduler runs on the writer's own connection, which is also\n // its gate: no writer (mode `off`) means no scheduler, and a writer that\n // has not created the file yet -- or failed soft and deleted it -- makes\n // cycles skip rather than fail, so a late-appearing file is picked up\n // without a restart.\n this.#purgeScheduler =\n opts.sqliteChangeLogPurge && this.#changeLogWriter\n ? new SQLiteChangeLogPurgeScheduler(\n lc,\n () => this.#changeLogWriter?.connection,\n () => this.#forwarder.getAcks(),\n opts.sqliteChangeLogPurge,\n )\n : undefined;\n this.#acker = new UpstreamAcker({\n trackPgChangeLog: true, // TODO: set false when retiring PG\n trackBackup: backupConfig?.litestreamVersion === 'v5',\n });\n const replicaSource = opts.sqliteChangeLogCompare\n ? replicaInitializationSource(lc, opts.sqliteChangeLogCompare.replicaFile)\n : undefined;\n this.#initializer = new ChangeLogInitializer(\n lc,\n {\n initFromPgChangeLog: true, // TODO: set false when retiring PG\n initFromReplica: replicaSource !== undefined,\n },\n {\n pgChangeLog: () =>\n this.#storer.getStartStreamInitializationParameters(),\n // Only reached when `initFromReplica` is set, i.e. when the option\n // that supplies the file is present.\n replica: () => must(replicaSource)(),\n // Use the Postgres resume point when it is present. Otherwise, let the\n // SQLite change log supply its own resume point.\n reconcileChangeLog: (resumeFrom, seed) =>\n resumeFrom\n ? this.#changeLogWriter?.reconcile(resumeFrom)\n : this.#changeLogWriter?.reconcileFromLog(seed),\n changeLog: () => this.#changeLogWriter?.connection,\n },\n );\n this.#sqliteCatchupOptions = opts.sqliteCatchup\n ? {\n ...opts.sqliteCatchup,\n // The real cleanup guard, in place of the catchup's no-op default:\n // registration and purge batches now serialize on one mutex.\n cleanupGuard:\n opts.sqliteCatchup.cleanupGuard ??\n this.#purgeScheduler?.cleanupGuard,\n }\n : undefined;\n this.#purgeLock = initialPurgeLock;\n this.#autoReset = autoReset;\n this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n this.#latestStatus = {tag: 'status'};\n }\n\n async run() {\n this.#lc.info?.('starting change stream');\n\n this.#forwarder.startProgressMonitor();\n\n const lagReportInit = await this.#source.startLagReporter();\n if (lagReportInit) {\n this.#latestStatus.lagReport = {\n nextSendTimeMs: lagReportInit.nextSendTimeMs,\n };\n // Record the commit time of the initiated lag report (i.e. \"head\")\n // for the purpose of skipping over any lag reports that are re-streamed\n // by the change-source in the case of a change-streamer starting from\n // an older watermark.\n this.#latestLagReportCommitTimeMs = lagReportInit.firstCommitTimeMs;\n }\n\n // Once this change-streamer acquires \"ownership\" of the change DB,\n // it is safe to start the storer.\n await this.#storer.assumeOwnership(this.#purgeLock);\n this.#purgeLock = null;\n\n // The threshold in (estimated number of) bytes to send() on subscriber\n // websockets before `await`-ing the I/O buffers to be ready for more.\n const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n while (this.#state.shouldRun()) {\n let err: unknown;\n let watermark: string \| null = null;\n let unflushedBytes = 0;\n try {\n // Initialization reconciles the change log for every stream\n // connection. It completes before `startStream`, so no change can\n // arrive during reconciliation.\n const {lastWatermark, backfillRequests} =\n await this.#initializer.initialize();\n // SQLite catchup must not be eligible until this has been initialized\n // from the durable PG head. Commits observed only since process startup\n // are insufficient after a change-streamer restart.\n this.#lastForwardedCommitWatermark = lastWatermark;\n const stream = await this.#source.startStream(\n lastWatermark,\n backfillRequests,\n );\n this.#storer.run().catch(e => stream.changes.cancel(e));\n\n this.#stream = stream;\n if (\n this.#state.resetBackoff() >\n REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n ) {\n // After recovering from a backoff for which a replication status\n // error was published, publish an OK status\n this.#replicationStatusPublisher.publish(\n this.#lc,\n 'Replicating',\n `Replicating from ${lastWatermark}`,\n );\n }\n watermark = null;\n\n this.#acker.reset(stream.acks);\n for await (const change of stream.changes) {\n this.#acker.trackDownstream(change);\n\n const [type, msg] = change;\n switch (type) {\n case 'status':\n if (\n msg.lagReport &&\n msg.lagReport.lastTimings.commitTimeMs >=\n this.#latestLagReportCommitTimeMs\n ) {\n // Lag reports are not stored in the cdc change log, but rather\n // only forwarded on \"live\" connections. When a new subscriber\n // is catching up, it is initialized with the #latestStatus\n // from which it can measure lag while catching up.\n this.#latestStatus.lagReport = msg.lagReport;\n this.#latestLagReportCommitTimeMs =\n msg.lagReport.lastTimings.commitTimeMs;\n this.#forwarder.sendStatus(this.#latestStatus);\n }\n continue;\n case 'control':\n await this.#handleControlMessage(msg);\n continue; // control messages are not stored/forwarded\n case 'begin':\n watermark = change[2].commitWatermark;\n break;\n case 'commit':\n if (watermark !== change[2].watermark) {\n throw new UnrecoverableError(\n `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n );\n }\n this.#txCounter.add(1);\n break;\n default:\n if (type === 'data') {\n this.#changeCounter.add(1);\n }\n if (watermark === null) {\n throw new UnrecoverableError(\n `${type} change (${msg.tag}) received before 'begin' message`,\n );\n }\n break;\n }\n\n const json = this.#storer.store(watermark, change);\n // The SQLite change log commits at transaction boundaries, and its\n // commit for this transaction lands here -- before the forward of the\n // `commit` message, and before #recordForwardedTransactionBoundary\n // advances what #captureRequiredHead reads. No `await` separates it\n // from #storer.store() above, which is the assertable form of\n // invariant 1: the storer only enqueues, and its Postgres commit\n // cannot complete without yielding, so a synchronous SQLite commit in\n // the same loop iteration always precedes anything that can advance\n // the watermark this stream would resume from. Never throws; a write\n // failure disables the writer rather than stopping replication.\n this.#changeLogWriter?.write(change, json);\n const entry: WatermarkedChange = [watermark, change[1].tag, json];\n unflushedBytes += json.length;\n if (unflushedBytes < flushBytesThreshold) {\n // pipeline changes until flushBytesThreshold\n this.#forwarder.forward(entry);\n this.#recordForwardedTransactionBoundary(type, entry[0]);\n } else {\n // Wait for messages to clear socket buffers to ensure that they\n // make their way to subscribers. Without this `await`, the\n // messages end up being buffered in this process, which:\n // (1) results in memory pressure and increased GC activity\n // (2) prevents subscribers from processing the messages as they\n // arrive, instead getting them in a large batch after being\n // idle while they were queued (causing further delays).\n const forwarded = this.#forwarder.forwardWithFlowControl(entry);\n // forwardWithFlowControl synchronously sends the entry and updates\n // the Forwarder's transaction state before returning its flow-\n // control promise. Record the boundary before awaiting that promise\n // so registrations during the wait observe the forwarded state.\n this.#recordForwardedTransactionBoundary(type, entry[0]);\n await stream.changes.doneOr(forwarded);\n unflushedBytes = 0;\n }\n\n if (type === 'commit' \|\| type === 'rollback') {\n watermark = null;\n }\n\n // Allow the storer to exert back pressure.\n const readyForMore = this.#storer.readyForMore();\n if (readyForMore) {\n await stream.changes.doneOr(readyForMore);\n }\n }\n } catch (e) {\n err = e;\n } finally {\n this.#stream?.changes.cancel();\n this.#stream = undefined;\n }\n\n // When the change stream is interrupted, abort any pending transaction.\n if (watermark) {\n this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n this.#storer.abort();\n // Rolling back the log leaves no rows for the interrupted transaction,\n // so the next connection's reconciliation sees a head at or below its\n // resume watermark rather than a partial transaction.\n this.#changeLogWriter?.abort();\n // A rollback ends the log's open transaction without a commit\n // notification; wake any purge batch waiting for that window.\n this.#purgeScheduler?.onWriterIdle();\n this.#forwarder.forward([watermark, 'rollback', ROLLBACK_JSON]);\n this.#recordForwardedTransactionBoundary('rollback', watermark);\n }\n\n // Backoff and drain any pending entries in the storer before reconnecting.\n await Promise.all([\n this.#storer.stop(),\n this.#state.backoff(this.#lc, err),\n this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n ? publishCriticalEvent(\n this.#lc,\n replicationStatusError(this.#lc, 'Replicating', err),\n )\n : promiseVoid,\n ]);\n }\n\n this.#forwarder.stopProgressMonitor();\n this.#lc.info?.('ChangeStreamer stopped');\n }\n\n async #handleControlMessage(msg: ChangeStreamControl[1]) {\n this.#lc.info?.('received control message', msg);\n const {tag} = msg;\n\n switch (tag) {\n case 'reset-required':\n await markResetRequired(this.#changeDB, this.#shard);\n await publishReplicationError(\n this.#lc,\n 'Replicating',\n msg.message ?? 'Resync required',\n msg.errorDetails,\n );\n if (this.#autoReset) {\n this.#lc.warn?.('shutting down for auto-reset');\n await this.stop(new AutoResetSignal());\n }\n break;\n default:\n unreachable(tag);\n }\n }\n\n async subscribe(ctx: SubscriberContext): Promise<Source<string>> {\n const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n if (mode === 'serving') {\n this.#serving.resolve();\n }\n let cleanupSubscriber = () => {};\n const downstream = Subscription.create<string>({\n cleanup: () => cleanupSubscriber(),\n });\n // No subscriber's ACK advances the SQLite change log's head any more: the\n // writer runs in this process, so the barrier is notified from the commit\n // itself (see #changeLogWriter's onCommit).\n const subscriber = new Subscriber(\n protocolVersion,\n id,\n watermark,\n downstream,\n () => this.#latestStatus,\n {},\n );\n const lc = this.#lc.withContext('subscriber', subscriber.id);\n cleanupSubscriber = () => {\n lc.info?.(`removing subscriber ${subscriber.id}`);\n this.#forwarder.remove(subscriber);\n };\n if (replicaVersion !== this.#replicaVersion) {\n lc.warn?.(`rejecting subscriber at replica version ${replicaVersion}`);\n subscriber.close(\n ErrorType.WrongReplicaVersion,\n `current replica version is ${\n this.#replicaVersion\n } (requested ${replicaVersion})`,\n );\n } else {\n lc.info?.(`adding subscriber ${subscriber.id}`);\n\n const sqliteCatchup = this.#selectSQLiteCatchup(ctx);\n if (sqliteCatchup) {\n cleanupSubscriber = () => sqliteCatchup.remove(subscriber);\n await sqliteCatchup.catchup(subscriber, () =>\n this.#captureRequiredHead(),\n );\n } else {\n // Keep the existing PG registration/catchup lockstep unchanged when\n // SQLite was not selected before Forwarder.add().\n this.#forwarder.add(subscriber);\n this.#storer.catchup(subscriber, mode);\n }\n }\n // Any snapshot reservation held by this task can be closed now that\n // it is subscribed to the change stream.\n this.#reservations?.close(ctx.taskID);\n return downstream;\n }\n\n async startSnapshotReservation(\n taskID: string,\n ): Promise<Source<SnapshotMessage>> {\n if (!this.#reservations) {\n throw new Error('backups are not configured');\n }\n const downstream = this.#reservations.open(taskID);\n\n try {\n // Wait for an in-flight SQLite purge batch before reading and\n // advertising the reservation's bounds.\n await (this.#purgeScheduler?.pause(taskID) ?? promiseVoid);\n // If a backup has been confirmed, immediately confirm the reservation.\n await this.#confirmReservations();\n return downstream;\n } catch (e) {\n // Cancel the reservation this call opened, not whatever currently\n // holds the task's slot: an overlapping retry for the same taskID may\n // have already replaced it, and a by-taskID close would tear down the\n // replacement and release its purge pause while it is advertising\n // snapshot bounds. cancel() routes through the subscription's cleanup,\n // which closes the reservation only if this instance still owns it.\n downstream.cancel();\n throw e;\n }\n }\n\n trackBackupWatermark(watermark: string) {\n this.#backupWatermark = watermark;\n this.#acker.trackBackup(watermark);\n // The durable backup floor is an independent input to each change-log\n // implementation. SQLite receives it even when the PG log has already\n // reached this watermark.\n this.#requestSQLitePurge('deferred');\n this.#maybeSchedulePGPurge();\n this.#maybeScheduleSQLitePurge();\n\n // Confirm any waiting reservations now that a backup has been confirmed.\n // Note that this is asynchronous and best effort; if it fails, the watermark\n // is still \"tracked\" and the confirmation will be retried on the next backup.\n void this.#confirmReservations().catch(e =>\n this.#lc.warn?.(`error confirming snapshot reservation`, e),\n );\n }\n\n async #confirmReservations() {\n if (this.#backupWatermark && this.#reservations?.confirmationsRequired()) {\n const {replicaVersion, minWatermark} = await this.#getChangeLogState();\n if (minWatermark <= this.#backupWatermark) {\n this.#reservations?.confirm(replicaVersion, this.#backupWatermark);\n } else {\n // Something is wrong here ... the change-log should not have been\n // purged past the backup watermark. If we confirm the reservation,\n // we may not be able to catch up from the backup that the requester\n // restores.\n this.#lc.error?.(\n `change-log minWatermark ${minWatermark} is later than backupWatermark ${this.#backupWatermark}. ` +\n `Delaying confirmation of snapshot reservation until next backup.`,\n );\n }\n }\n }\n\n #maybeSchedulePGPurge(): void {\n const backupWatermark = this.#backupWatermark;\n if (\n this.#pgPurgeScheduled \|\|\n this.#pgPurgeRunning \|\|\n backupWatermark === undefined \|\|\n this.#pgPurgedWatermark >= backupWatermark\n ) {\n return;\n }\n this.#pgPurgeScheduled = true;\n this.#state.setTimeout(() => {\n this.#pgPurgeScheduled = false;\n this.#pgPurgeRunning = true;\n return this.#purgePGChangeLog().finally(() => {\n this.#pgPurgeRunning = false;\n this.#maybeSchedulePGPurge();\n });\n }, CLEANUP_DELAY_MS);\n }\n\n #maybeScheduleSQLitePurge(): void {\n if (\n this.#sqlitePurgeScheduled \|\|\n this.#sqlitePurgeRunning \|\|\n this.#backupWatermark === undefined \|\|\n this.#sqlitePurgeContinuation === undefined\n ) {\n return;\n }\n const delay =\n this.#sqlitePurgeContinuation === 'immediate' ? 0 : CLEANUP_DELAY_MS;\n this.#sqlitePurgeScheduled = true;\n this.#state.setTimeout(() => {\n this.#sqlitePurgeScheduled = false;\n this.#sqlitePurgeRunning = true;\n return this.#purgeSQLiteChangeLog().finally(() => {\n this.#sqlitePurgeRunning = false;\n this.#maybeScheduleSQLitePurge();\n });\n }, delay);\n }\n\n async #getChangeLogState(): Promise<{\n replicaVersion: string;\n minWatermark: string;\n }> {\n const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n if (!minWatermark) {\n this.#lc.warn?.(\n `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n );\n }\n return {\n replicaVersion: this.#replicaVersion,\n minWatermark: minWatermark ?? this.#replicaVersion,\n };\n }\n\n #getCleanupFloor(): {\n backupWatermark: string;\n purgeWatermark: string;\n current: string[];\n } {\n const backupWatermark = this.#backupWatermark;\n assert(\n backupWatermark !== undefined,\n 'cleanup cannot run without a backup watermark',\n );\n const current = [\n ...this.#forwarder.getAcks(),\n ...(this.#reservations?.getReservedWatermarks() ?? []),\n ];\n // The cleanup delay above is the grace period for disconnected\n // subscribers to reconnect and expose their ACKs. Once it expires, an\n // empty set places no additional constraint on the confirmed backup\n // watermark and must not pin either change log indefinitely.\n return {\n backupWatermark,\n purgeWatermark: min(backupWatermark, ...current),\n current,\n };\n }\n\n async #purgePGChangeLog(): Promise<void> {\n try {\n const {backupWatermark, purgeWatermark, current} =\n this.#getCleanupFloor();\n if (purgeWatermark < backupWatermark) {\n if (this.#loggedBehindWatermark !== purgeWatermark) {\n this.#loggedBehindWatermark = purgeWatermark;\n this.#lc.info?.(\n `At least one client is behind backup ${backupWatermark}`,\n {watermarks: current},\n );\n }\n } else {\n this.#loggedBehindWatermark = undefined;\n }\n if (purgeWatermark <= this.#pgPurgedWatermark) {\n return;\n }\n this.#lc.info?.(`Purging PG changes before ${purgeWatermark} ...`);\n const start = performance.now();\n const deleted = await this.#storer.purgeRecordsBefore(purgeWatermark);\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `Purged ${deleted} PG changes before ${purgeWatermark} (${elapsed} ms)`,\n );\n this.#pgPurgedWatermark = purgeWatermark;\n } catch (e) {\n this.#lc.warn?.(`error purging the PG change log`, e);\n }\n }\n\n async #purgeSQLiteChangeLog(): Promise<void> {\n const scheduler = this.#purgeScheduler;\n if (!scheduler) {\n return;\n }\n try {\n const {backupWatermark, purgeWatermark} = this.#getCleanupFloor();\n // Consume the request before starting. A backup notification that\n // arrives during this pass records another request, which is merged with\n // the continuation returned by this pass rather than being overwritten.\n this.#sqlitePurgeContinuation = undefined;\n if (purgeWatermark < backupWatermark) {\n // Live constraint changes are not edge-triggered. Keep evaluating the\n // floor until it reaches the durable backup watermark, independently\n // of whether the PG implementation still exists.\n this.#requestSQLitePurge('deferred');\n }\n const result = await scheduler.purge(purgeWatermark);\n this.#requestSQLitePurge(result.continuation);\n } catch (e) {\n // purge() is fail-soft, but retain the coordinator's retry if an\n // unexpected caller-boundary failure escapes it.\n this.#requestSQLitePurge('deferred');\n this.#lc.warn?.(`error purging the SQLite change log`, e);\n }\n }\n\n #requestSQLitePurge(continuation: PurgeContinuation \| undefined): void {\n if (!this.#purgeScheduler \|\| continuation === undefined) {\n return;\n }\n if (\n continuation === 'immediate' \|\|\n this.#sqlitePurgeContinuation === undefined\n ) {\n this.#sqlitePurgeContinuation = continuation;\n }\n }\n\n async stop(err?: unknown) {\n this.#state.stop(this.#lc, err);\n this.#stream?.changes.cancel();\n this.#purgeScheduler?.stop();\n this.#sqliteCatchup?.close();\n this.#changeLogWriter?.close();\n await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);\n }\n\n #recordForwardedTransactionBoundary(\n type: ChangeStreamData[0],\n watermark: string,\n ) {\n switch (type) {\n case 'begin':\n assert(\n this.#currentTransactionCompletion === undefined,\n 'forwarded begin while a transaction is already in progress',\n );\n this.#currentTransactionCompletion =\n resolver<ForwardedTransactionCompletion>();\n this.#transactionForwardStartedAt = performance.now();\n break;\n case 'commit': {\n const completion = this.#currentTransactionCompletion;\n assert(completion, 'forwarded commit without a pending transaction');\n this.#recordTransactionForwardDuration();\n this.#lastForwardedCommitWatermark = watermark;\n this.#currentTransactionCompletion = undefined;\n completion.resolve({kind: 'committed', watermark});\n break;\n }\n case 'rollback': {\n const completion = this.#currentTransactionCompletion;\n assert(completion, 'forwarded rollback without a pending transaction');\n const committed = this.#lastForwardedCommitWatermark;\n assert(committed, 'last forwarded commit watermark is not initialized');\n this.#transactionForwardStartedAt = undefined;\n this.#currentTransactionCompletion = undefined;\n completion.resolve({kind: 'rolled-back', watermark: committed});\n break;\n }\n }\n }\n\n #recordTransactionForwardDuration() {\n const startedAt = this.#transactionForwardStartedAt;\n this.#transactionForwardStartedAt = undefined;\n if (startedAt !== undefined) {\n this.#transactionForwardDuration.recordMs(performance.now() - startedAt, {\n sqlite_change_log: this.#changeLogWriter?.enabled ? 'on' : 'off',\n });\n }\n }\n\n /\n Closes and forgets the cached catchup coordinator, so that a later\n * subscription re-opens the log from scratch rather than reading a handle\n * whose file is gone.\n /\n #closeSQLiteCatchup() {\n this.#sqliteCatchup?.close();\n this.#sqliteCatchup = undefined;\n }\n\n #captureRequiredHead(): string \| Promise<string> {\n const committed = this.#lastForwardedCommitWatermark;\n assert(committed, 'last forwarded commit watermark is not initialized');\n return (\n this.#currentTransactionCompletion?.promise.then(\n completion => completion.watermark,\n ) ?? committed\n );\n }\n\n #selectSQLiteCatchup(\n ctx: SubscriberContext,\n ): SQLiteChangeLogCatchup \| undefined {\n const opts = this.#sqliteCatchupOptions;\n // Slice 11 supplies a non-default selector. Until then, this keeps all\n // production subscriptions on PG even though the complete SQLite handoff\n // path is wired and testable.\n if (this.#lastForwardedCommitWatermark === undefined \|\| !opts) {\n return undefined;\n }\n // SQLite catchup is only for disposable serving replicas. Backup\n // subscribers retain the existing PG recovery policy until RMv2 can\n // resume the canonical replica directly from replica/backup/slot state.\n // Enforce this before the selector so no canary policy can override it.\n if (ctx.mode !== 'serving') {\n this.#lc.info?.(\n `not serving backup subscriber ${ctx.id} from SQLite catchup`,\n );\n return undefined;\n }\n if (ctx.logsChangeStream) {\n // The reason for this exclusion is gone: the writer is no longer a\n // subscriber, so nothing can wait on its own ACK, and no replicator this\n // version ships sets the parameter. It stays until slice 11 lifts it\n // deliberately, because a subscriber that predates the writer's move does\n // set it, and serving one from SQLite is a change in behavior rather than\n // a cleanup.\n this.#lc.warn?.(\n `not serving legacy change-log writer ${ctx.id} from SQLite catchup`,\n );\n return undefined;\n }\n if (!opts.shouldUse?.(ctx)) {\n return undefined;\n }\n return this.#sqliteCatchup ?? this.#openSQLiteCatchup(opts, ctx);\n }\n\n /\n Opens the change log and, if it can serve, wraps it in the coordinator that\n * subsequent subscriptions reuse.\n \n The log may not exist yet -- the writer creates it at the stream loop's\n * first reconcile, and deletes it when it fails soft -- or may exist without\n * content, or may be unreadable. None of those can be allowed to fail a\n * subscription:\n * this is the last point at which PG catchup is still available -- past\n * `Forwarder.add()` the subscriber is committed to SQLite -- so each declines\n * here instead. Neither the failure nor the reader is retained, so a later\n * subscription retries from scratch.\n /\n #openSQLiteCatchup(\n opts: SQLiteCatchupOptions,\n ctx: SubscriberContext,\n ): SQLiteChangeLogCatchup \| undefined {\n let reader: SQLiteChangeLogReader \| undefined;\n try {\n reader = new SQLiteChangeLogReader(this.#lc, opts.changeLogFile);\n // `plan()` doubles as the readiness check: it reports `not-ready` when\n // the writer has not created or seeded the stream table. The barrier\n // cannot wait its way out of that, since a log with no head can never\n // reach the required one.\n if (reader.plan(ctx.watermark).kind === 'not-ready') {\n reader.close();\n this.#declineSQLiteCatchup(\n ctx,\n opts,\n `${opts.changeLogFile} has no changes to serve yet`,\n );\n return undefined;\n }\n } catch (e) {\n // An absent file is the common case, since a readonly handle cannot\n // create one. A corrupt or truncated file lands here too, and is equally\n // not a reason to fail the subscription.\n reader?.close();\n this.#declineSQLiteCatchup(\n ctx,\n opts,\n `cannot read ${opts.changeLogFile}`,\n e,\n );\n return undefined;\n }\n this.#changeLogUnavailableSince = undefined;\n // Readiness is checked once, on the way to the cached coordinator: a log\n // with content keeps it. Purging preserves the latest transaction as a\n // catchup boundary, and `reconcileChangeLog` reseeds inside a single\n // transaction, so a reader never observes an emptied log.\n this.#sqliteCatchup = new SQLiteChangeLogCatchup(\n this.#lc,\n this.#forwarder,\n reader,\n {\n batchSize: opts.readBatchRows,\n barrierTimeoutMs: opts.barrierTimeoutMs,\n barrierPollIntervalMs: opts.barrierPollIntervalMs,\n cleanupGuard: opts.cleanupGuard,\n },\n );\n return this.#sqliteCatchup;\n }\n\n /\n Reports falling back to PG catchup, tracking how long the log has been\n * unavailable so that a subscription that merely arrived before the writer's\n * first reconcile does not look like an incident.\n /\n #declineSQLiteCatchup(\n ctx: SubscriberContext,\n opts: SQLiteCatchupOptions,\n reason: string,\n error?: unknown,\n ): void {\n const now = Date.now();\n this.#changeLogUnavailableSince ??= now;\n const unavailableMs = now - this.#changeLogUnavailableSince;\n const threshold =\n opts.notReadyWarnThresholdMs ??\n DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS;\n this.#lc[unavailableMs >= threshold ? 'warn' : 'debug']?.(\n `serving ${ctx.id} from PG catchup: ${reason} ` +\n `(unavailable for ${unavailableMs} ms)`,\n ...(error === undefined ? [] : [error]),\n );\n }\n}\n\ntype ForwardedTransactionCompletion =\n \| {kind: 'committed'; watermark: string}\n \| {kind: 'rolled-back'; watermark: string};\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n// `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS 3;\n\nconst ROLLBACK_JSON = JSON.stringify([\n 'rollback',\n {tag: 'rollback'},\n] satisfies Rollback);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwJA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,cACA,WACA,WACA,MACA,eAAe,YACiB;CAChC,MAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,aAAa,KAAA,GACb,YACF;CAEA,MAAM,EAAC,mBAAkB;CACzB,OAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,cACA,WACA,WACA,MACA,YACF;AACF;AAEA,IAAM,8CAA8C;AAQpD,IAAM,mDAAmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4IzD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CAUA,WAAoB,SAAS;CAE7B,aAAsB,mBACpB,eACA,gBACA,kCACF;CACA,iBAA0B,mBACxB,eACA,WACA,qDACF;CAIA,8BAAuC,4BACrC,eACA,gCACA,yIAEF;CAEA;CACA,+BAA+B;CAC/B;CACA,qBAA6B;;;;;;;CAO7B;CACA;CACA;CAGA,oBAAoB;CACpB,kBAAkB;CAClB,wBAAwB;CACxB,sBAAsB;CACtB;CACA;CACA;CACA;CACA;CACA;CAIA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,cACA,kBACA,WACA,MACA,eAAe,YACf;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EACxD,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,UAAU,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,KAAKO,OAAO,iBAAiB,SAAS,GAAG,SAAS,IAC9D,QAAO,KAAK,KAAK,GAAG,GACpB,IACF;EACA,KAAKN,aAAa,IAAI,UAAU,IAAI;GAClC,uCACE,KAAK;GACP,wCACE,KAAK;EACT,CAAC;EACD,KAAKC,gBAAgB,eACjB,IAAI,qBAAqB,IAAI,eAAc,WACzC,KAAKI,iBAAiB,OAAO,MAAM,CACrC,IACA,KAAA;EACJ,KAAKH,8BAA8B;EACnC,KAAKE,mBAAmB,KAAK,wBACzB,IAAI,sBAAsB,IAAI;GAC5B,GAAG,KAAK;GACR,WAAU,cAAa;IACrB,KAAKU,gBAAgB,kBAAkB,SAAS;IAGhD,KAAKT,iBAAiB,aAAa;GACrC;GAIA,kBAAkB,KAAKU,oBAAoB;GAC3C,iBAAiB,KAAKA,oBAAoB;EAC5C,CAAC,IACD,KAAA;EAMJ,KAAKV,kBACH,KAAK,wBAAwB,KAAKD,mBAC9B,IAAI,8BACF,UACM,KAAKA,kBAAkB,kBACvB,KAAKJ,WAAW,QAAQ,GAC9B,KAAK,oBACP,IACA,KAAA;EACN,KAAKM,SAAS,IAAI,cAAc;GAC9B,kBAAkB;GAClB,aAAa,cAAc,sBAAsB;EACnD,CAAC;EACD,MAAM,gBAAgB,KAAK,yBACvB,4BAA4B,IAAI,KAAK,uBAAuB,WAAW,IACvE,KAAA;EACJ,KAAKC,eAAe,IAAI,qBACtB,IACA;GACE,qBAAqB;GACrB,iBAAiB,kBAAkB,KAAA;EACrC,GACA;GACE,mBACE,KAAKR,QAAQ,uCAAuC;GAGtD,eAAe,KAAK,aAAa,EAAE;GAGnC,qBAAqB,YAAY,SAC/B,aACI,KAAKK,kBAAkB,UAAU,UAAU,IAC3C,KAAKA,kBAAkB,iBAAiB,IAAI;GAClD,iBAAiB,KAAKA,kBAAkB;EAC1C,CACF;EACA,KAAKD,wBAAwB,KAAK,gBAC9B;GACE,GAAG,KAAK;GAGR,cACE,KAAK,cAAc,gBACnB,KAAKE,iBAAiB;EAC1B,IACA,KAAA;EACJ,KAAKW,aAAa;EAClB,KAAKR,aAAa;EAClB,KAAKC,SAAS,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,YAAY;EAC/D,KAAKQ,gBAAgB,EAAC,KAAK,SAAQ;CACrC;CAEA,MAAM,MAAM;EACV,KAAKvB,IAAI,OAAO,wBAAwB;EAExC,KAAKM,WAAW,qBAAqB;EAErC,MAAM,gBAAgB,MAAM,KAAKF,QAAQ,iBAAiB;EAC1D,IAAI,eAAe;GACjB,KAAKmB,cAAc,YAAY,EAC7B,gBAAgB,cAAc,eAChC;GAKA,KAAKC,+BAA+B,cAAc;EACpD;EAIA,MAAM,KAAKnB,QAAQ,gBAAgB,KAAKiB,UAAU;EAClD,KAAKA,aAAa;EAIlB,MAAM,sBAAsB,wBAAwB,KAAK;EAEzD,OAAO,KAAKP,OAAO,UAAU,GAAG;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;GACrB,IAAI;IAIF,MAAM,EAAC,eAAe,qBACpB,MAAM,KAAKF,aAAa,WAAW;IAIrC,KAAKY,gCAAgC;IACrC,MAAM,SAAS,MAAM,KAAKrB,QAAQ,YAChC,eACA,gBACF;IACA,KAAKC,QAAQ,IAAI,EAAE,OAAM,MAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;IAEtD,KAAKqB,UAAU;IACf,IACE,KAAKX,OAAO,aAAa,IACzB,6CAIA,KAAKP,4BAA4B,QAC/B,KAAKR,KACL,eACA,oBAAoB,eACtB;IAEF,YAAY;IAEZ,KAAKY,OAAO,MAAM,OAAO,IAAI;IAC7B,WAAW,MAAM,UAAU,OAAO,SAAS;KACzC,KAAKA,OAAO,gBAAgB,MAAM;KAElC,MAAM,CAAC,MAAM,OAAO;KACpB,QAAQ,MAAR;MACE,KAAK;OACH,IACE,IAAI,aACJ,IAAI,UAAU,YAAY,gBACxB,KAAKY,8BACP;QAKA,KAAKD,cAAc,YAAY,IAAI;QACnC,KAAKC,+BACH,IAAI,UAAU,YAAY;QAC5B,KAAKlB,WAAW,WAAW,KAAKiB,aAAa;OAC/C;OACA;MACF,KAAK;OACH,MAAM,KAAKI,sBAAsB,GAAG;OACpC;MACF,KAAK;OACH,YAAY,OAAO,GAAG;OACtB;MACF,KAAK;OACH,IAAI,cAAc,OAAO,GAAG,WAC1B,MAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,WAC9E;OAEF,KAAKV,WAAW,IAAI,CAAC;OACrB;MACF;OACE,IAAI,SAAS,QACX,KAAKC,eAAe,IAAI,CAAC;OAE3B,IAAI,cAAc,MAChB,MAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,kCAC7B;OAEF;KACJ;KAEA,MAAM,OAAO,KAAKb,QAAQ,MAAM,WAAW,MAAM;KAWjD,KAAKK,kBAAkB,MAAM,QAAQ,IAAI;KACzC,MAAM,QAA2B;MAAC;MAAW,OAAO,GAAG;MAAK;KAAI;KAChE,kBAAkB,KAAK;KACvB,IAAI,iBAAiB,qBAAqB;MAExC,KAAKJ,WAAW,QAAQ,KAAK;MAC7B,KAAKsB,oCAAoC,MAAM,MAAM,EAAE;KACzD,OAAO;MAQL,MAAM,YAAY,KAAKtB,WAAW,uBAAuB,KAAK;MAK9D,KAAKsB,oCAAoC,MAAM,MAAM,EAAE;MACvD,MAAM,OAAO,QAAQ,OAAO,SAAS;MACrC,iBAAiB;KACnB;KAEA,IAAI,SAAS,YAAY,SAAS,YAChC,YAAY;KAId,MAAM,eAAe,KAAKvB,QAAQ,aAAa;KAC/C,IAAI,cACF,MAAM,OAAO,QAAQ,OAAO,YAAY;IAE5C;GACF,SAAS,GAAG;IACV,MAAM;GACR,UAAU;IACR,KAAKqB,SAAS,QAAQ,OAAO;IAC7B,KAAKA,UAAU,KAAA;GACjB;GAGA,IAAI,WAAW;IACb,KAAK1B,IAAI,OAAO,oCAAoC,WAAW;IAC/D,KAAKK,QAAQ,MAAM;IAInB,KAAKK,kBAAkB,MAAM;IAG7B,KAAKC,iBAAiB,aAAa;IACnC,KAAKL,WAAW,QAAQ;KAAC;KAAW;KAAY;IAAa,CAAC;IAC9D,KAAKsB,oCAAoC,YAAY,SAAS;GAChE;GAGA,MAAM,QAAQ,IAAI;IAChB,KAAKvB,QAAQ,KAAK;IAClB,KAAKU,OAAO,QAAQ,KAAKf,KAAK,GAAG;IACjC,KAAKe,OAAO,aAAa,8CACrB,qBACE,KAAKf,KACL,uBAAuB,KAAKA,KAAK,eAAe,GAAG,CACrD,IACA;GACN,CAAC;EACH;EAEA,KAAKM,WAAW,oBAAoB;EACpC,KAAKN,IAAI,OAAO,wBAAwB;CAC1C;CAEA,MAAM2B,sBAAsB,KAA6B;EACvD,KAAK3B,IAAI,OAAO,4BAA4B,GAAG;EAC/C,MAAM,EAAC,QAAO;EAEd,QAAQ,KAAR;GACE,KAAK;IACH,MAAM,kBAAkB,KAAKE,WAAW,KAAKD,MAAM;IACnD,MAAM,wBACJ,KAAKD,KACL,eACA,IAAI,WAAW,mBACf,IAAI,YACN;IACA,IAAI,KAAKc,YAAY;KACnB,KAAKd,IAAI,OAAO,8BAA8B;KAC9C,MAAM,KAAK,KAAK,IAAI,gBAAgB,CAAC;IACvC;IACA;GACF,SACE,YAAY,GAAG;EACnB;CACF;CAEA,MAAM,UAAU,KAAiD;EAC/D,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;EAC/D,IAAI,SAAS,WACX,KAAKgB,SAAS,QAAQ;EAExB,IAAI,0BAA0B,CAAC;EAC/B,MAAM,aAAa,aAAa,OAAe,EAC7C,eAAe,kBAAkB,EACnC,CAAC;EAID,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,KAAKO,eACX,CAAC,CACH;EACA,MAAM,KAAK,KAAKvB,IAAI,YAAY,cAAc,WAAW,EAAE;EAC3D,0BAA0B;GACxB,GAAG,OAAO,uBAAuB,WAAW,IAAI;GAChD,KAAKM,WAAW,OAAO,UAAU;EACnC;EACA,IAAI,mBAAmB,KAAKH,iBAAiB;GAC3C,GAAG,OAAO,2CAA2C,gBAAgB;GACrE,WAAW,MACT,GACA,8BACE,KAAKA,gBACN,cAAc,eAAe,EAChC;EACF,OAAO;GACL,GAAG,OAAO,qBAAqB,WAAW,IAAI;GAE9C,MAAM,gBAAgB,KAAK0B,qBAAqB,GAAG;GACnD,IAAI,eAAe;IACjB,0BAA0B,cAAc,OAAO,UAAU;IACzD,MAAM,cAAc,QAAQ,kBAC1B,KAAKC,qBAAqB,CAC5B;GACF,OAAO;IAGL,KAAKxB,WAAW,IAAI,UAAU;IAC9B,KAAKD,QAAQ,QAAQ,YAAY,IAAI;GACvC;EACF;EAGA,KAAKE,eAAe,MAAM,IAAI,MAAM;EACpC,OAAO;CACT;CAEA,MAAM,yBACJ,QACkC;EAClC,IAAI,CAAC,KAAKA,eACR,MAAM,IAAI,MAAM,4BAA4B;EAE9C,MAAM,aAAa,KAAKA,cAAc,KAAK,MAAM;EAEjD,IAAI;GAGF,OAAO,KAAKI,iBAAiB,MAAM,MAAM,KAAK;GAE9C,MAAM,KAAKoB,qBAAqB;GAChC,OAAO;EACT,SAAS,GAAG;GAOV,WAAW,OAAO;GAClB,MAAM;EACR;CACF;CAEA,qBAAqB,WAAmB;EACtC,KAAKC,mBAAmB;EACxB,KAAKpB,OAAO,YAAY,SAAS;EAIjC,KAAKqB,oBAAoB,UAAU;EACnC,KAAKC,sBAAsB;EAC3B,KAAKC,0BAA0B;EAK/B,KAAUJ,qBAAqB,EAAE,OAAM,MACrC,KAAK/B,IAAI,OAAO,yCAAyC,CAAC,CAC5D;CACF;CAEA,MAAM+B,uBAAuB;EAC3B,IAAI,KAAKC,oBAAoB,KAAKzB,eAAe,sBAAsB,GAAG;GACxE,MAAM,EAAC,gBAAgB,iBAAgB,MAAM,KAAK6B,mBAAmB;GACrE,IAAI,gBAAgB,KAAKJ,kBACvB,KAAKzB,eAAe,QAAQ,gBAAgB,KAAKyB,gBAAgB;QAMjE,KAAKhC,IAAI,QACP,2BAA2B,aAAa,iCAAiC,KAAKgC,iBAAiB,mEAEjG;EAEJ;CACF;CAEA,wBAA8B;EAC5B,MAAM,kBAAkB,KAAKA;EAC7B,IACE,KAAKK,qBACL,KAAKC,mBACL,oBAAoB,KAAA,KACpB,KAAKC,sBAAsB,iBAE3B;EAEF,KAAKF,oBAAoB;EACzB,KAAKtB,OAAO,iBAAiB;GAC3B,KAAKsB,oBAAoB;GACzB,KAAKC,kBAAkB;GACvB,OAAO,KAAKE,kBAAkB,EAAE,cAAc;IAC5C,KAAKF,kBAAkB;IACvB,KAAKJ,sBAAsB;GAC7B,CAAC;EACH,GAAG,gBAAgB;CACrB;CAEA,4BAAkC;EAChC,IACE,KAAKO,yBACL,KAAKC,uBACL,KAAKV,qBAAqB,KAAA,KAC1B,KAAKW,6BAA6B,KAAA,GAElC;EAEF,MAAM,QACJ,KAAKA,6BAA6B,cAAc,IAAI;EACtD,KAAKF,wBAAwB;EAC7B,KAAK1B,OAAO,iBAAiB;GAC3B,KAAK0B,wBAAwB;GAC7B,KAAKC,sBAAsB;GAC3B,OAAO,KAAKE,sBAAsB,EAAE,cAAc;IAChD,KAAKF,sBAAsB;IAC3B,KAAKP,0BAA0B;GACjC,CAAC;EACH,GAAG,KAAK;CACV;CAEA,MAAMC,qBAGH;EACD,MAAM,eAAe,MAAM,KAAK/B,QAAQ,0BAA0B;EAClE,IAAI,CAAC,cACH,KAAKL,IAAI,OACP,8EACF;EAEF,OAAO;GACL,gBAAgB,KAAKG;GACrB,cAAc,gBAAgB,KAAKA;EACrC;CACF;CAEA,mBAIE;EACA,MAAM,kBAAkB,KAAK6B;EAC7B,OACE,oBAAoB,KAAA,GACpB,+CACF;EACA,MAAM,UAAU,CACd,GAAG,KAAK1B,WAAW,QAAQ,GAC3B,GAAI,KAAKC,eAAe,sBAAsB,KAAK,CAAC,CACtD;EAKA,OAAO;GACL;GACA,gBAAgB,IAAI,iBAAiB,GAAG,OAAO;GAC/C;EACF;CACF;CAEA,MAAMiC,oBAAmC;EACvC,IAAI;GACF,MAAM,EAAC,iBAAiB,gBAAgB,YACtC,KAAKK,iBAAiB;GACxB,IAAI,iBAAiB;QACf,KAAKC,2BAA2B,gBAAgB;KAClD,KAAKA,yBAAyB;KAC9B,KAAK9C,IAAI,OACP,wCAAwC,mBACxC,EAAC,YAAY,QAAO,CACtB;IACF;UAEA,KAAK8C,yBAAyB,KAAA;GAEhC,IAAI,kBAAkB,KAAKP,oBACzB;GAEF,KAAKvC,IAAI,OAAO,6BAA6B,eAAe,KAAK;GACjE,MAAM,QAAQ,YAAY,IAAI;GAC9B,MAAM,UAAU,MAAM,KAAKK,QAAQ,mBAAmB,cAAc;GACpE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;GACrD,KAAKL,IAAI,OACP,UAAU,QAAQ,qBAAqB,eAAe,IAAI,QAAQ,KACpE;GACA,KAAKuC,qBAAqB;EAC5B,SAAS,GAAG;GACV,KAAKvC,IAAI,OAAO,mCAAmC,CAAC;EACtD;CACF;CAEA,MAAM4C,wBAAuC;EAC3C,MAAM,YAAY,KAAKjC;EACvB,IAAI,CAAC,WACH;EAEF,IAAI;GACF,MAAM,EAAC,iBAAiB,mBAAkB,KAAKkC,iBAAiB;GAIhE,KAAKF,2BAA2B,KAAA;GAChC,IAAI,iBAAiB,iBAInB,KAAKV,oBAAoB,UAAU;GAErC,MAAM,SAAS,MAAM,UAAU,MAAM,cAAc;GACnD,KAAKA,oBAAoB,OAAO,YAAY;EAC9C,SAAS,GAAG;GAGV,KAAKA,oBAAoB,UAAU;GACnC,KAAKjC,IAAI,OAAO,uCAAuC,CAAC;EAC1D;CACF;CAEA,oBAAoB,cAAmD;EACrE,IAAI,CAAC,KAAKW,mBAAmB,iBAAiB,KAAA,GAC5C;EAEF,IACE,iBAAiB,eACjB,KAAKgC,6BAA6B,KAAA,GAElC,KAAKA,2BAA2B;CAEpC;CAEA,MAAM,KAAK,KAAe;EACxB,KAAK5B,OAAO,KAAK,KAAKf,KAAK,GAAG;EAC9B,KAAK0B,SAAS,QAAQ,OAAO;EAC7B,KAAKf,iBAAiB,KAAK;EAC3B,KAAKS,gBAAgB,MAAM;EAC3B,KAAKV,kBAAkB,MAAM;EAC7B,MAAM,QAAQ,WAAW,CAAC,KAAKL,QAAQ,KAAK,GAAG,KAAKD,QAAQ,KAAK,CAAC,CAAC;CACrE;CAEA,oCACE,MACA,WACA;EACA,QAAQ,MAAR;GACE,KAAK;IACH,OACE,KAAK2C,kCAAkC,KAAA,GACvC,4DACF;IACA,KAAKA,gCACH,SAAyC;IAC3C,KAAKC,+BAA+B,YAAY,IAAI;IACpD;GACF,KAAK,UAAU;IACb,MAAM,aAAa,KAAKD;IACxB,OAAO,YAAY,gDAAgD;IACnE,KAAKE,kCAAkC;IACvC,KAAKxB,gCAAgC;IACrC,KAAKsB,gCAAgC,KAAA;IACrC,WAAW,QAAQ;KAAC,MAAM;KAAa;IAAS,CAAC;IACjD;GACF;GACA,KAAK,YAAY;IACf,MAAM,aAAa,KAAKA;IACxB,OAAO,YAAY,kDAAkD;IACrE,MAAM,YAAY,KAAKtB;IACvB,OAAO,WAAW,oDAAoD;IACtE,KAAKuB,+BAA+B,KAAA;IACpC,KAAKD,gCAAgC,KAAA;IACrC,WAAW,QAAQ;KAAC,MAAM;KAAe,WAAW;IAAS,CAAC;IAC9D;GACF;EACF;CACF;CAEA,oCAAoC;EAClC,MAAM,YAAY,KAAKC;EACvB,KAAKA,+BAA+B,KAAA;EACpC,IAAI,cAAc,KAAA,GAChB,KAAK7B,4BAA4B,SAAS,YAAY,IAAI,IAAI,WAAW,EACvE,mBAAmB,KAAKT,kBAAkB,UAAU,OAAO,MAC7D,CAAC;CAEL;;;;;;CAOA,sBAAsB;EACpB,KAAKU,gBAAgB,MAAM;EAC3B,KAAKA,iBAAiB,KAAA;CACxB;CAEA,uBAAiD;EAC/C,MAAM,YAAY,KAAKK;EACvB,OAAO,WAAW,oDAAoD;EACtE,OACE,KAAKsB,+BAA+B,QAAQ,MAC1C,eAAc,WAAW,SAC3B,KAAK;CAET;CAEA,qBACE,KACoC;EACpC,MAAM,OAAO,KAAKtC;EAIlB,IAAI,KAAKgB,kCAAkC,KAAA,KAAa,CAAC,MACvD;EAMF,IAAI,IAAI,SAAS,WAAW;GAC1B,KAAKzB,IAAI,OACP,iCAAiC,IAAI,GAAG,qBAC1C;GACA;EACF;EACA,IAAI,IAAI,kBAAkB;GAOxB,KAAKA,IAAI,OACP,wCAAwC,IAAI,GAAG,qBACjD;GACA;EACF;EACA,IAAI,CAAC,KAAK,YAAY,GAAG,GACvB;EAEF,OAAO,KAAKoB,kBAAkB,KAAK8B,mBAAmB,MAAM,GAAG;CACjE;;;;;;;;;;;;;;CAeA,mBACE,MACA,KACoC;EACpC,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,sBAAsB,KAAKlD,KAAK,KAAK,aAAa;GAK/D,IAAI,OAAO,KAAK,IAAI,SAAS,EAAE,SAAS,aAAa;IACnD,OAAO,MAAM;IACb,KAAKmD,sBACH,KACA,MACA,GAAG,KAAK,cAAc,6BACxB;IACA;GACF;EACF,SAAS,GAAG;GAIV,QAAQ,MAAM;GACd,KAAKA,sBACH,KACA,MACA,eAAe,KAAK,iBACpB,CACF;GACA;EACF;EACA,KAAKC,6BAA6B,KAAA;EAKlC,KAAKhC,iBAAiB,IAAI,uBACxB,KAAKpB,KACL,KAAKM,YACL,QACA;GACE,WAAW,KAAK;GAChB,kBAAkB,KAAK;GACvB,uBAAuB,KAAK;GAC5B,cAAc,KAAK;EACrB,CACF;EACA,OAAO,KAAKc;CACd;;;;;;CAOA,sBACE,KACA,MACA,QACA,OACM;EACN,MAAM,MAAM,KAAK,IAAI;EACrB,KAAKgC,+BAA+B;EACpC,MAAM,gBAAgB,MAAM,KAAKA;EACjC,MAAM,YACJ,KAAK,2BACL;EACF,KAAKpD,IAAI,iBAAiB,YAAY,SAAS,WAC7C,WAAW,IAAI,GAAG,oBAAoB,OAAO,oBACvB,cAAc,OACpC,GAAI,UAAU,KAAA,IAAY,CAAC,IAAI,CAAC,KAAK,CACvC;CACF;AACF;AAqBA,IAAM,mBAAmB,6BAA6B;AAEtD,IAAM,gBAAgB,KAAK,UAAU,CACnC,YACA,EAAC,KAAK,WAAU,CAClB,CAAoB"}
1	+ {"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDBProvider","#replicaVersion","#source","#storer","#forwarder","#reservations","#replicationStatusPublisher","#sqliteCatchupOptions","#changeLogWriter","#purgeScheduler","#comparator","#acker","#initializer","#readRouter","#autoReset","#state","#serving","#txCounter","#changeCounter","#transactionForwardDuration","#catchupRoutes","#reservationDemotions","#reservationConfirmDelays","#sqliteCatchup","#closeSQLiteCatchup","#purgeLock","#latestStatus","#latestLagReportCommitTimeMs","#lastForwardedCommitWatermark","#stream","#handleControlMessage","#recordForwardedTransactionBoundary","#selectSQLiteCatchup","#captureRequiredHead","#recordCatchupRoute","#confirmReservations","#backupWatermark","#requestSQLitePurge","#maybeSchedulePGPurge","#maybeScheduleSQLitePurge","#getChangeLogState","#pgPurgeScheduled","#pgPurgeRunning","#pgPurgedWatermark","#purgePGChangeLog","#sqlitePurgeScheduled","#sqlitePurgeRunning","#sqlitePurgeContinuation","#purgeSQLiteChangeLog","#getCleanupFloor","#loggedBehindWatermark","#currentTransactionCompletion","#transactionForwardStartedAt","#recordTransactionForwardDuration","#declineSQLiteCatchup","#openSQLiteCatchup","#changeLogUnavailableSince"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import {getDefaultHighWaterMark} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseOrAbort} from '../../../../shared/src/promise-race.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {\n getOrCreateCounter,\n getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {min} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n ChangeSource,\n ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n type ChangeStreamControl,\n type ChangeStreamData,\n type Rollback,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {LitestreamVersion} from '../litestream/metrics.ts';\nimport {\n publishReplicationError,\n replicationStatusError,\n type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n DEFAULT_MAX_RETRY_DELAY_MS,\n RunningState,\n UnrecoverableError,\n} from '../running-state.ts';\nimport {\n ChangeLogInitializer,\n replicaInitializationSource,\n} from './change-log-initializer.ts';\nimport {\n type ChangeStreamerService,\n type Status,\n type SubscriberContext,\n type WatermarkedChange,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {\n AutoResetSignal,\n ensureReplicationConfig,\n markResetRequired,\n} from './schema/tables.ts';\nimport {SnapshotReservations} from './snapshot-reservations.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\nimport {\n SQLiteChangeLogCatchup,\n type SQLiteChangeLogCleanupGuard,\n} from './sqlite-change-log-catchup.ts';\nimport {\n SQLiteChangeLogComparator,\n type SQLiteChangeLogCompareOptions,\n} from './sqlite-change-log-comparator.ts';\nimport {\n SQLiteChangeLogPurgeScheduler,\n type PurgeContinuation,\n type SQLiteChangeLogPurgeSchedulerOptions,\n} from './sqlite-change-log-purge-scheduler.ts';\nimport {\n inspectSQLiteChangeLog,\n SQLiteChangeLogReadRouter,\n type ChangeLogReadRoute,\n type SQLiteChangeLogCoverage,\n} from './sqlite-change-log-read-router.ts';\nimport {SQLiteChangeLogReader} from './sqlite-change-log-reader.ts';\nimport {\n SQLiteChangeLogWriter,\n type SQLiteChangeLogWriterOptions,\n} from './sqlite-change-log-writer.ts';\nimport type {PostgresDBProvider} from './storer.ts';\nimport {\n Storer,\n type PurgeLock,\n type TuningOptions as StorerOptions,\n} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\nimport {UpstreamAcker} from './upstream-acker.ts';\n\nexport type BackupConfig = {\n backupURL: string;\n litestreamVersion: LitestreamVersion;\n};\n\nexport type SQLiteCatchupOptions = {\n /** The change-log database, i.e. `changeLogFileName(replicaFile)`. /\n changeLogFile: string;\n readBatchRows: number;\n barrierTimeoutMs: number;\n /\n Backstop for the change-log writer's ACK, which is what normally releases\n * the barrier. Defaults to the catchup coordinator's interval.\n /\n barrierPollIntervalMs?: number \| undefined;\n /\n Optional policy hook used by focused tests. Production canary selection is\n * supplied by `sqliteChangeLogServe` and is stable by shard plus task ID.\n * Backup subscribers are rejected before either selector is invoked.\n /\n shouldUse?: ((ctx: SubscriberContext) => boolean) \| undefined;\n /\n Overrides the purge scheduler's guard, for tests. When absent, the\n * service supplies the scheduler's real writer-serialized guard, or the\n * catchup's no-op default when purging is not configured.\n /\n cleanupGuard?: SQLiteChangeLogCleanupGuard \| undefined;\n /\n How long the change log may be unavailable before declining to serve from\n * it is reported as a warning rather than a debug line. Defaults to\n * {@link DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS}.\n /\n notReadyWarnThresholdMs?: number \| undefined;\n};\n\nexport type SQLiteChangeLogServeOptions = {\n /* Stable percentage of eligible serving tasks routed to SQLite. /\n readPercent: number;\n /\n Stable percentage of eligible serving tasks routed to a log that has not\n * yet aged through {@link retentionMs}. Zero keeps a reseeded log on PG for\n * that whole window; above zero it serves that share of tasks, and a\n * reservation it cannot cover is demoted rather than held pending.\n * Defaults to zero.\n /\n coldReadPercent?: number \| undefined;\n /* The warm-up window a reseeded log ages through. /\n retentionMs: number;\n /* Injectable for deterministic breaker tests. /\n failureCooldownMs?: number \| undefined;\n /* Injectable for deterministic warm-up and breaker tests. /\n now?: (() => number) \| undefined;\n /* Injectable readiness inspection for service-level tests. /\n inspect?: (() => SQLiteChangeLogCoverage \| undefined) \| undefined;\n};\n\nexport type TuningOptions = StorerOptions & {\n flowControlConsensusTimeoutProportion: number;\n flowControlSlowSubscriberGracePeriodMs?: number \| undefined;\n sqliteCatchup?: SQLiteCatchupOptions \| undefined;\n /\n Supplied when `sqliteChangeLogMode != off`, i.e. this is the gate on the\n * change-log writer. Absent, nothing writes the log.\n /\n sqliteChangeLogWriter?:\n \| Omit<\n SQLiteChangeLogWriterOptions,\n 'onCommit' \| 'onDisabled' \| 'onRebuilt'\n >\n \| undefined;\n /\n Also supplied when `sqliteChangeLogMode != off`, and deliberately not\n * gated on the read path: in `write` mode no reader ever opens, and this is\n * the configuration the purge scheduler actually ships in. The scheduler\n * runs on the writer's own connection, so with the writer absent (mode\n * `off`) it is never constructed, and a leftover file on disk is left to\n * the replicator's cleanup.\n /\n sqliteChangeLogPurge?: SQLiteChangeLogPurgeSchedulerOptions \| undefined;\n /\n Supplied in `compare` mode and later modes.\n * `replicaFile` enables the initialization comparison.\n * The other fields configure sampled catchup comparisons.\n * Both checks are advisory, and Postgres remains authoritative.\n * The comparator also requires the writer and catchup options.\n /\n sqliteChangeLogCompare?:\n \| (SQLiteChangeLogCompareOptions & {replicaFile: string})\n \| undefined;\n /* Supplied only in `serve` mode. A zero percentage keeps every read on PG. /\n sqliteChangeLogServe?: SQLiteChangeLogServeOptions \| undefined;\n};\n\n/\n Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n /\nexport async function initializeStreamer(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n changeSource: ChangeSource,\n replicationStatusPublisher: ReplicationStatusPublisher,\n subscriptionState: SubscriptionState,\n backupConfig: BackupConfig \| null,\n purgeLock: PurgeLock \| null,\n autoReset: boolean,\n opts: TuningOptions,\n setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n await ensureReplicationConfig(\n lc,\n changeDB,\n subscriptionState,\n shard,\n autoReset,\n purgeLock ?? undefined,\n setTimeoutFn,\n );\n\n // Dynamically creates connection pools that the implementation uses to\n // isolate concurrent, long-running transactions. This works around a bug in\n // the postgres.js client where connections get swapped in certain conditions.\n //\n // https://github.com/porsager/postgres/issues/1204\n const changeDBProvider: PostgresDBProvider = (\n applicationName: string,\n max: number,\n ) =>\n postgres(\n defu(\n {max, connection: {['application_name']: applicationName}},\n // ParsedOptions are technically compatible with Options, but happen\n // to not be typed that way. The postgres.js author does an equivalent\n // merge of ParsedOptions and Options here:\n // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n changeDB.options as unknown as Options<Record<string, PostgresType>>,\n ),\n ) as PostgresDB;\n\n const {replicaVersion} = subscriptionState;\n return new ChangeStreamerImpl(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDBProvider,\n replicaVersion,\n changeSource,\n replicationStatusPublisher,\n backupConfig,\n purgeLock,\n autoReset,\n opts,\n setTimeoutFn,\n );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\n// How long the change log may be unavailable before declining to serve from it\n// stops looking like normal startup ordering. Subscriptions can arrive before\n// the stream loop's first reconcile has created and seeded the log, so early\n// declines are expected. Still declining a minute later means the log is not\n// being written at all -- `sqliteChangeLogMode=off`, a writer that has failed\n// soft, or a path mismatch -- which is worth surfacing.\nconst DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS = 60_000;\n\n/\n Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n \n ### Subscriber Catchup\n \n Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n \n \n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * \| Historic changes in storage \| Pending (streamed) tx \| Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * Replication stream\n * > > > > > > > > >\n * ^ ---> required catchup ---> ^\n * Subscriber watermark Subscription begins\n * ```\n \n Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n \n Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n \n * If the streamer is in the middle of a pending Transaction, the\n * Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n * state, new changes are not forwarded to the Subscriber, and catchup\n * is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n * by the Storer, it begins catchup on the Subscriber (with a READONLY\n * snapshot so that it does not block subsequent storage operations).\n * This catchup is thus guaranteed to load the change log entries of\n * that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n * Subscriber from the \"queued\" to the \"active\" set of clients such that\n * the Subscriber begins receiving new changes, starting from the next\n * Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n * is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n * and the Subscriber henceforth forwards changes as they are received.\n \n In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n \n ### Watermarks and ordering\n \n The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n \n https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n \n > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n \n In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n \n A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n \n ```\n * LSN: 1 2 3 4 5 6 7 8 9 10\n * tx1: begin data data data commit\n * tx2: begin data data data commit\n * ```\n \n will arrive as:\n \n ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n \n Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n \n Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n \n * A `commit` message is assigned to a watermark corresponding to its LSN.\n * These are guaranteed to be in commit order by definition.\n \n * `begin` and `data` messages are assigned to the watermark of the\n * preceding `commit` (the previous transaction, or the replication\n * slot's starting LSN) plus 1. This guarantees that they will be sorted\n * after the previously commit transaction even if their LSNs came before it.\n * This is referred to as the `preCommitWatermark`.\n \n * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n * the position of the message within its transaction, with the `begin` message\n * starting at `0`. This guarantees that `begin` and `data` messages will be\n * fetched in the original ChangeSource order during catchup.\n \n `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n \n ### Cleanup\n \n As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n \n More concretely:\n \n * The `initial`, backup-derived watermark is the earliest to which cleanup\n * should ever happen.\n \n * However, it is possible for the replica backup to be ahead of a connected\n * subscriber; and if a network error causes that subscriber to retry from its\n * last watermark, the change streamer must support it.\n \n Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n /\nclass ChangeStreamerImpl implements ChangeStreamerService {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #changeDBProvider: PostgresDBProvider;\n readonly #replicaVersion: string;\n readonly #source: ChangeSource;\n readonly #storer: Storer;\n readonly #forwarder: Forwarder;\n readonly #reservations: SnapshotReservations \| undefined;\n readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n readonly #sqliteCatchupOptions: SQLiteCatchupOptions \| undefined;\n readonly #changeLogWriter: SQLiteChangeLogWriter \| undefined;\n readonly #purgeScheduler: SQLiteChangeLogPurgeScheduler \| undefined;\n readonly #comparator: SQLiteChangeLogComparator \| undefined;\n readonly #acker: UpstreamAcker;\n readonly #initializer: ChangeLogInitializer;\n readonly #readRouter: SQLiteChangeLogReadRouter \| undefined;\n\n readonly #autoReset: boolean;\n readonly #state: RunningState;\n\n // Starting the (Postgres) ChangeStream results in killing the previous\n // Postgres subscriber, potentially creating a gap in which the old\n // change-streamer has shut down and the new change-streamer has not yet\n // been recognized as \"healthy\" (and thus does not get any requests).\n //\n // To minimize this gap, delay starting the ChangeStream until the first\n // request from a `serving` replicator, indicating that higher level\n // load-balancing / routing logic has begun routing requests to this task.\n readonly #serving = resolver();\n\n readonly #txCounter = getOrCreateCounter(\n 'replication',\n 'transactions',\n 'Count of replicated transactions',\n );\n readonly #changeCounter = getOrCreateCounter(\n 'replication',\n 'changes',\n 'Count of replicated changes (DML or DDL statements)',\n );\n // The number the SQLite change-log commit lands in. It is labeled by whether\n // the log is being written so that the cost of putting a commit on the\n // forward path is attributable rather than inferred.\n readonly #transactionForwardDuration = getOrCreateLatencyHistogram(\n 'replication',\n 'transaction_forward_duration',\n \"Time from receiving a transaction's `begin` to forwarding its `commit`, \" +\n 'i.e. the change-streamer half of forward-to-subscriber latency.',\n );\n readonly #catchupRoutes = getOrCreateCounter(\n 'replication',\n 'sqlite_change_log.catchup_routes',\n 'Catchup subscriptions by selected source and low-cardinality reason.',\n );\n readonly #reservationDemotions = getOrCreateCounter(\n 'replication',\n 'sqlite_change_log.reservation_demotions',\n 'Snapshot reservations demoted from SQLite to PG because the change ' +\n 'log could not cover the backup being restored. With PG retired ' +\n 'these followers have no fallback, so this is the rate at which one ' +\n 'would instead have to wait for a later backup.',\n );\n readonly #reservationConfirmDelays = getOrCreateCounter(\n 'replication',\n 'sqlite_change_log.reservation_confirm_delays',\n 'Snapshot reservations whose confirmation was deferred because the ' +\n \"selected source's change-log minimum was later than the backup \" +\n 'watermark. Counted once per reservation, by that source.',\n );\n\n #latestStatus: Status;\n #latestLagReportCommitTimeMs = 0;\n #backupWatermark: string \| undefined;\n #pgPurgedWatermark: string = '';\n /\n The floor last logged as held back by a laggard. The level-triggered\n * retry re-evaluates every {@link CLEANUP_DELAY_MS}; logging only when the\n * blocking floor moves keeps a stuck subscriber from emitting the same\n * line indefinitely.\n /\n #loggedBehindWatermark: string \| undefined;\n #sqlitePurgeContinuation: PurgeContinuation \| undefined;\n #purgeLock: PurgeLock \| null;\n // PG and SQLite intentionally own separate level-triggered loops. Neither\n // waits for, advances, or retries the other.\n #pgPurgeScheduled = false;\n #pgPurgeRunning = false;\n #sqlitePurgeScheduled = false;\n #sqlitePurgeRunning = false;\n #stream: ChangeStream \| undefined;\n #sqliteCatchup: SQLiteChangeLogCatchup \| undefined;\n #changeLogUnavailableSince: number \| undefined;\n #lastForwardedCommitWatermark: string \| undefined;\n #transactionForwardStartedAt: number \| undefined;\n #currentTransactionCompletion:\n \| Resolver<ForwardedTransactionCompletion>\n \| undefined;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDBProvider: PostgresDBProvider,\n replicaVersion: string,\n source: ChangeSource,\n replicationStatusPublisher: ReplicationStatusPublisher,\n backupConfig: BackupConfig \| null,\n initialPurgeLock: PurgeLock \| null,\n autoReset: boolean,\n opts: TuningOptions,\n setTimeoutFn = setTimeout,\n ) {\n this.id = `change-streamer`;\n this.#lc = lc.withContext('component', 'change-streamer');\n this.#shard = shard;\n this.#changeDBProvider = changeDBProvider;\n this.#replicaVersion = replicaVersion;\n this.#source = source;\n this.#storer = new Storer(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDBProvider,\n replicaVersion,\n consumed => this.#acker.trackPgChangeLog(consumed[2].watermark),\n err => this.stop(err),\n opts,\n );\n this.#forwarder = new Forwarder(lc, {\n flowControlConsensusTimeoutProportion:\n opts.flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodMs:\n opts.flowControlSlowSubscriberGracePeriodMs,\n });\n const serveOptions = opts.sqliteChangeLogServe;\n const writerOptions = opts.sqliteChangeLogWriter;\n const catchupOptions = opts.sqliteCatchup;\n this.#readRouter =\n serveOptions && writerOptions && catchupOptions\n ? new SQLiteChangeLogReadRouter({\n shard,\n readPercent: serveOptions.readPercent,\n coldReadPercent: serveOptions.coldReadPercent,\n retentionMs: serveOptions.retentionMs,\n failureCooldownMs: serveOptions.failureCooldownMs,\n now: serveOptions.now,\n inspect:\n serveOptions.inspect ??\n (() =>\n inspectSQLiteChangeLog(\n lc,\n catchupOptions.changeLogFile,\n writerOptions.identity,\n )),\n })\n : undefined;\n this.#reservations = backupConfig\n ? new SnapshotReservations(lc, backupConfig, taskID => {\n this.#readRouter?.release(taskID);\n this.#purgeScheduler?.resume(taskID);\n })\n : undefined;\n this.#replicationStatusPublisher = replicationStatusPublisher;\n this.#changeLogWriter = opts.sqliteChangeLogWriter\n ? new SQLiteChangeLogWriter(lc, {\n ...opts.sqliteChangeLogWriter,\n onCommit: watermark => {\n this.#sqliteCatchup?.onChangeLogCommit(watermark);\n // The commit closed the log's transaction, i.e. opened a purge\n // window (§3.3).\n this.#purgeScheduler?.onWriterIdle();\n },\n // Fail-soft deletes the file, and the reader is cached here, so it\n // has to go with it: otherwise it serves an unlinked inode while\n // every new open sees nothing.\n onDisabled: () => {\n // The writer stays disabled and the file stays absent for the life\n // of this process, so unlike a transient read/barrier failure this\n // breaker never expires.\n this.#readRouter?.trip(true);\n this.#closeSQLiteCatchup();\n this.#comparator?.stop();\n },\n onRebuilt: () => {\n this.#closeSQLiteCatchup();\n // Invalidate cycles that can still read the replaced file.\n this.#comparator?.invalidate();\n },\n })\n : undefined;\n // The purge scheduler runs on the writer's own connection, which is also\n // its gate: no writer (mode `off`) means no scheduler, and a writer that\n // has not created the file yet -- or failed soft and deleted it -- makes\n // cycles skip rather than fail, so a late-appearing file is picked up\n // without a restart.\n this.#purgeScheduler =\n opts.sqliteChangeLogPurge && this.#changeLogWriter\n ? new SQLiteChangeLogPurgeScheduler(\n lc,\n () => this.#changeLogWriter?.connection,\n () => this.#forwarder.getAcks(),\n opts.sqliteChangeLogPurge,\n )\n : undefined;\n this.#acker = new UpstreamAcker({\n trackPgChangeLog: true, // TODO: set false when retiring PG\n trackBackup: backupConfig?.litestreamVersion === 'v5',\n });\n const replicaSource = opts.sqliteChangeLogCompare\n ? replicaInitializationSource(lc, opts.sqliteChangeLogCompare.replicaFile)\n : undefined;\n this.#initializer = new ChangeLogInitializer(\n lc,\n {\n initFromPgChangeLog: true, // TODO: set false when retiring PG\n initFromReplica: replicaSource !== undefined,\n },\n {\n pgChangeLog: () =>\n this.#storer.getStartStreamInitializationParameters(),\n // Only reached when `initFromReplica` is set, i.e. when the option\n // that supplies the file is present.\n replica: () => must(replicaSource)(),\n // Use the Postgres resume point when it is present. Otherwise, let the\n // SQLite change log supply its own resume point.\n reconcileChangeLog: (resumeFrom, seed) =>\n resumeFrom\n ? this.#changeLogWriter?.reconcile(resumeFrom)\n : this.#changeLogWriter?.reconcileFromLog(seed),\n changeLog: () => this.#changeLogWriter?.connection,\n },\n );\n this.#sqliteCatchupOptions = opts.sqliteCatchup\n ? {\n ...opts.sqliteCatchup,\n // The real cleanup guard, in place of the catchup's no-op default:\n // registration and purge batches now serialize on one mutex.\n cleanupGuard:\n opts.sqliteCatchup.cleanupGuard ??\n this.#purgeScheduler?.cleanupGuard,\n }\n : undefined;\n // Compare mode requires the writer and catchup configuration.\n this.#comparator =\n opts.sqliteChangeLogCompare &&\n opts.sqliteChangeLogWriter &&\n opts.sqliteCatchup\n ? new SQLiteChangeLogComparator(\n lc,\n shard,\n opts.sqliteCatchup.changeLogFile,\n opts.sqliteChangeLogWriter.identity,\n this.#storer,\n {setTimeoutFn, ...opts.sqliteChangeLogCompare},\n )\n : undefined;\n this.#purgeLock = initialPurgeLock;\n this.#autoReset = autoReset;\n this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n this.#latestStatus = {tag: 'status'};\n }\n\n async run() {\n this.#lc.info?.('starting change stream');\n\n this.#forwarder.startProgressMonitor();\n\n const lagReportInit = await this.#source.startLagReporter();\n if (lagReportInit) {\n this.#latestStatus.lagReport = {\n nextSendTimeMs: lagReportInit.nextSendTimeMs,\n };\n // Record the commit time of the initiated lag report (i.e. \"head\")\n // for the purpose of skipping over any lag reports that are re-streamed\n // by the change-source in the case of a change-streamer starting from\n // an older watermark.\n this.#latestLagReportCommitTimeMs = lagReportInit.firstCommitTimeMs;\n }\n\n // Once this change-streamer acquires \"ownership\" of the change DB,\n // it is safe to start the storer.\n await this.#storer.assumeOwnership(this.#purgeLock);\n this.#purgeLock = null;\n\n // The threshold in (estimated number of) bytes to send() on subscriber\n // websockets before `await`-ing the I/O buffers to be ready for more.\n const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n while (this.#state.shouldRun()) {\n let err: unknown;\n let watermark: string \| null = null;\n let unflushedBytes = 0;\n try {\n // Initialization reconciles the change log for every stream\n // connection. It completes before `startStream`, so no change can\n // arrive during reconciliation.\n const {lastWatermark, backfillRequests} =\n await this.#initializer.initialize();\n // SQLite catchup must not be eligible until this has been initialized\n // from the durable PG head. Commits observed only since process startup\n // are insufficient after a change-streamer restart.\n this.#lastForwardedCommitWatermark = lastWatermark;\n const stream = await this.#source.startStream(\n lastWatermark,\n backfillRequests,\n );\n this.#storer.run().catch(e => stream.changes.cancel(e));\n\n this.#stream = stream;\n if (\n this.#state.resetBackoff() >\n REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n ) {\n // After recovering from a backoff for which a replication status\n // error was published, publish an OK status\n this.#replicationStatusPublisher.publish(\n this.#lc,\n 'Replicating',\n `Replicating from ${lastWatermark}`,\n );\n }\n watermark = null;\n\n this.#acker.reset(stream.acks);\n for await (const change of stream.changes) {\n this.#acker.trackDownstream(change);\n\n const [type, msg] = change;\n switch (type) {\n case 'status':\n if (\n msg.lagReport &&\n msg.lagReport.lastTimings.commitTimeMs >=\n this.#latestLagReportCommitTimeMs\n ) {\n // Lag reports are not stored in the cdc change log, but rather\n // only forwarded on \"live\" connections. When a new subscriber\n // is catching up, it is initialized with the #latestStatus\n // from which it can measure lag while catching up.\n this.#latestStatus.lagReport = msg.lagReport;\n this.#latestLagReportCommitTimeMs =\n msg.lagReport.lastTimings.commitTimeMs;\n this.#forwarder.sendStatus(this.#latestStatus);\n }\n continue;\n case 'control':\n await this.#handleControlMessage(msg);\n continue; // control messages are not stored/forwarded\n case 'begin':\n watermark = change[2].commitWatermark;\n break;\n case 'commit':\n if (watermark !== change[2].watermark) {\n throw new UnrecoverableError(\n `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n );\n }\n this.#txCounter.add(1);\n break;\n default:\n if (type === 'data') {\n this.#changeCounter.add(1);\n }\n if (watermark === null) {\n throw new UnrecoverableError(\n `${type} change (${msg.tag}) received before 'begin' message`,\n );\n }\n break;\n }\n\n const json = this.#storer.store(watermark, change);\n // The SQLite change log commits at transaction boundaries, and its\n // commit for this transaction lands here -- before the forward of the\n // `commit` message, and before #recordForwardedTransactionBoundary\n // advances what #captureRequiredHead reads. No `await` separates it\n // from #storer.store() above, which is the assertable form of\n // invariant 1: the storer only enqueues, and its Postgres commit\n // cannot complete without yielding, so a synchronous SQLite commit in\n // the same loop iteration always precedes anything that can advance\n // the watermark this stream would resume from. Never throws; a write\n // failure disables the writer rather than stopping replication.\n this.#changeLogWriter?.write(change, json);\n const entry: WatermarkedChange = [watermark, change[1].tag, json];\n unflushedBytes += json.length;\n if (unflushedBytes < flushBytesThreshold) {\n // pipeline changes until flushBytesThreshold\n this.#forwarder.forward(entry);\n this.#recordForwardedTransactionBoundary(type, entry[0]);\n } else {\n // Wait for messages to clear socket buffers to ensure that they\n // make their way to subscribers. Without this `await`, the\n // messages end up being buffered in this process, which:\n // (1) results in memory pressure and increased GC activity\n // (2) prevents subscribers from processing the messages as they\n // arrive, instead getting them in a large batch after being\n // idle while they were queued (causing further delays).\n const forwarded = this.#forwarder.forwardWithFlowControl(entry);\n // forwardWithFlowControl synchronously sends the entry and updates\n // the Forwarder's transaction state before returning its flow-\n // control promise. Record the boundary before awaiting that promise\n // so registrations during the wait observe the forwarded state.\n this.#recordForwardedTransactionBoundary(type, entry[0]);\n await promiseOrAbort(\n forwarded,\n stream.changes.signal,\n this.#state.signal,\n );\n unflushedBytes = 0;\n }\n\n if (type === 'commit' \|\| type === 'rollback') {\n watermark = null;\n }\n\n // Allow the storer to exert back pressure.\n const readyForMore = this.#storer.readyForMore();\n if (readyForMore) {\n await promiseOrAbort(\n readyForMore,\n stream.changes.signal,\n this.#state.signal,\n );\n }\n }\n } catch (e) {\n err = e;\n } finally {\n this.#stream?.changes.cancel();\n this.#stream = undefined;\n }\n\n // When the change stream is interrupted, abort any pending transaction.\n if (watermark) {\n this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n this.#storer.abort();\n // Rolling back the log leaves no rows for the interrupted transaction,\n // so the next connection's reconciliation sees a head at or below its\n // resume watermark rather than a partial transaction.\n this.#changeLogWriter?.abort();\n // A rollback ends the log's open transaction without a commit\n // notification; wake any purge batch waiting for that window.\n this.#purgeScheduler?.onWriterIdle();\n this.#forwarder.forward([watermark, 'rollback', ROLLBACK_JSON]);\n this.#recordForwardedTransactionBoundary('rollback', watermark);\n }\n\n // Backoff and drain any pending entries in the storer before reconnecting.\n await Promise.all([\n this.#storer.stop(),\n this.#state.backoff(this.#lc, err),\n this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n ? publishCriticalEvent(\n this.#lc,\n replicationStatusError(this.#lc, 'Replicating', err),\n )\n : promiseVoid,\n ]);\n }\n\n this.#forwarder.stopProgressMonitor();\n this.#lc.info?.('ChangeStreamer stopped');\n }\n\n async #handleControlMessage(msg: ChangeStreamControl[1]) {\n this.#lc.info?.('received control message', msg);\n const {tag} = msg;\n\n switch (tag) {\n case 'reset-required':\n await markResetRequired(\n this.#changeDBProvider('change-streamer-reset', 1),\n this.#shard,\n );\n await publishReplicationError(\n this.#lc,\n 'Replicating',\n msg.message ?? 'Resync required',\n msg.errorDetails,\n );\n if (this.#autoReset) {\n this.#lc.warn?.('shutting down for auto-reset');\n await this.stop(new AutoResetSignal());\n }\n break;\n default:\n unreachable(tag);\n }\n }\n\n async subscribe(ctx: SubscriberContext): Promise<Source<string>> {\n const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n if (mode === 'serving') {\n this.#serving.resolve();\n }\n let cleanupSubscriber = () => {};\n const downstream = Subscription.create<string>({\n cleanup: () => cleanupSubscriber(),\n });\n // No subscriber's ACK advances the SQLite change log's head any more: the\n // writer runs in this process, so the barrier is notified from the commit\n // itself (see #changeLogWriter's onCommit).\n const subscriber = new Subscriber(\n protocolVersion,\n id,\n mode,\n watermark,\n downstream,\n () => this.#latestStatus,\n {},\n );\n const lc = this.#lc.withContext('subscriber', subscriber.id);\n const removeFromForwarder = () => {\n lc.info?.(`removing subscriber ${subscriber.id}`);\n this.#forwarder.remove(subscriber);\n };\n cleanupSubscriber = removeFromForwarder;\n if (replicaVersion !== this.#replicaVersion) {\n lc.warn?.(`rejecting subscriber at replica version ${replicaVersion}`);\n subscriber.close(\n ErrorType.WrongReplicaVersion,\n `current replica version is ${\n this.#replicaVersion\n } (requested ${replicaVersion})`,\n );\n } else {\n lc.info?.(`adding subscriber ${subscriber.id}`);\n\n const catchupFromPG = () => {\n // Keep the existing PG registration/catchup lockstep unchanged when\n // SQLite was not selected before Forwarder.add().\n cleanupSubscriber = removeFromForwarder;\n this.#forwarder.add(subscriber);\n this.#storer.catchup(subscriber, mode);\n };\n const sqliteSelection = this.#selectSQLiteCatchup(lc, ctx);\n if (!sqliteSelection) {\n catchupFromPG();\n } else {\n const {catchup, reason, coverage, logWarm} = sqliteSelection;\n cleanupSubscriber = () => catchup.remove(subscriber);\n const registration = await catchup.catchup(\n subscriber,\n () => this.#captureRequiredHead(),\n {logWarm},\n );\n switch (registration.kind) {\n case 'registered':\n lc.debug?.(\n `serving ${ctx.id} from SQLite catchup`,\n ...(coverage ? [{sqliteChangeLogCoverage: coverage}] : []),\n );\n this.#recordCatchupRoute('sqlite', reason);\n break;\n case 'uncovered':\n lc.info?.(\n `serving ${ctx.id} from PG catchup: subscriber watermark ` +\n `${ctx.watermark} is below the SQLite change-log minimum ` +\n registration.minWatermark,\n );\n this.#recordCatchupRoute('pg', 'watermark-uncovered');\n catchupFromPG();\n break;\n case 'declined':\n // Registration failed before the subscriber was committed to\n // SQLite, so PG is still available. The coordinator has already\n // tripped the breaker, which keeps the retry off SQLite for the\n // cooldown; it is deliberately not closed here, since closing it\n // would abort the catchups of every other subscriber it is\n // serving.\n lc.error?.(\n `serving ${ctx.id} from PG catchup: SQLite catchup ` +\n `registration failed`,\n registration.error,\n );\n this.#recordCatchupRoute('pg', 'registration-failed');\n catchupFromPG();\n break;\n case 'handled':\n // The coordinator closed or failed the subscriber itself, so\n // there is nothing left to route. Still counted, so that the\n // route counter sums to the number of subscriptions.\n this.#recordCatchupRoute('sqlite', 'registration-handled');\n break;\n default:\n unreachable(registration);\n }\n }\n }\n // Any snapshot reservation held by this task can be closed now that\n // it is subscribed to the change stream.\n this.#reservations?.close(ctx.taskID);\n return downstream;\n }\n\n async startSnapshotReservation(\n taskID: string,\n ): Promise<Source<SnapshotMessage>> {\n if (!this.#reservations) {\n throw new Error('backups are not configured');\n }\n const downstream = this.#reservations.open(taskID);\n\n try {\n // Wait for an in-flight SQLite purge batch before reading and\n // advertising the reservation's bounds.\n await (this.#purgeScheduler?.pause(taskID) ?? promiseVoid);\n // A concurrent retry for this task may have superseded and cancelled\n // this reservation while its purge pause was settling. Only the current\n // owner may replace the task's source pin.\n if (!this.#reservations.isCurrent(taskID, downstream)) {\n return downstream;\n }\n // Pin after the purge pause has settled, so the SQLite minimum captured\n // by the router cannot move before it is advertised. #confirmReservations\n // skips an unpinned task while this await is in flight.\n this.#readRouter?.pin(taskID);\n // If a backup has been confirmed, immediately confirm the reservation.\n await this.#confirmReservations();\n return downstream;\n } catch (e) {\n // Cancel the reservation this call opened, not whatever currently\n // holds the task's slot: an overlapping retry for the same taskID may\n // have already replaced it, and a by-taskID close would tear down the\n // replacement and release its purge pause while it is advertising\n // snapshot bounds. cancel() routes through the subscription's cleanup,\n // which closes the reservation only if this instance still owns it.\n downstream.cancel();\n throw e;\n }\n }\n\n trackBackupWatermark(watermark: string) {\n this.#backupWatermark = watermark;\n this.#acker.trackBackup(watermark);\n // The durable backup floor is an independent input to each change-log\n // implementation. SQLite receives it even when the PG log has already\n // reached this watermark.\n this.#requestSQLitePurge('deferred');\n this.#maybeSchedulePGPurge();\n this.#maybeScheduleSQLitePurge();\n\n // Confirm any waiting reservations now that a backup has been confirmed.\n // Note that this is asynchronous and best effort; if it fails, the watermark\n // is still \"tracked\" and the confirmation will be retried on the next backup.\n void this.#confirmReservations().catch(e =>\n this.#lc.warn?.(`error confirming snapshot reservation`, e),\n );\n }\n\n async #confirmReservations() {\n const backupWatermark = this.#backupWatermark;\n const reservations = this.#reservations;\n if (\n backupWatermark === undefined \|\|\n !reservations?.confirmationsRequired()\n ) {\n return;\n }\n\n // Resolve PG bounds before touching any pin. Everything below runs to\n // completion without awaiting, which is what keeps a reservation's\n // advertised bounds and its pin in agreement: a concurrent /snapshot\n // retry for the same task replaces the reservation and re-pins it, and\n // confirming that replacement with the previous pin's bounds is exactly\n // the mismatch pinning exists to prevent. The cost is one PG round trip\n // per call even when every reservation is pinned to SQLite.\n const pgState = await this.#getChangeLogState();\n for (const taskID of reservations.unconfirmedTaskIDs()) {\n let route = this.#readRouter?.peek(taskID);\n // startSnapshotReservation pins only after an in-flight purge has\n // completed. A task with no pin yet is confirmed by that path instead.\n if (this.#readRouter && route === undefined) {\n continue;\n }\n\n if (route?.source === 'sqlite') {\n const coverage = must(\n route.coverage,\n 'a pinned SQLite route must carry its covered range',\n );\n if (coverage.minWatermark > backupWatermark) {\n // A log seeded after this backup, most often. Holding the\n // reservation until a backup reaches the log's minimum would stall\n // a follower that PG can serve now, so move it -- pin included.\n this.#lc.info?.(\n `demoting ${taskID} to PG catchup: SQLite change-log minimum ` +\n `${coverage.minWatermark} is later than backupWatermark ` +\n backupWatermark,\n );\n this.#reservationDemotions.add(1);\n route = must(this.#readRouter).demote(taskID);\n }\n }\n\n const source = route?.source ?? 'pg';\n let minWatermark: string;\n if (route?.source === 'sqlite') {\n // Demotion above already moved every SQLite route the backup is\n // outside of, so this one covers it and confirms below.\n minWatermark = must(\n route.coverage,\n 'a pinned SQLite route must carry its covered range',\n ).minWatermark;\n } else {\n ({minWatermark} = pgState);\n }\n\n if (minWatermark <= backupWatermark) {\n reservations.confirmFor(\n taskID,\n pgState.replicaVersion,\n backupWatermark,\n source,\n );\n } else {\n // PG cannot catch a restored replica up from this backup yet. Keep\n // the reservation pending until a later backup moves the durable\n // watermark into its covered range.\n if (reservations.noteConfirmationDelayed(taskID)) {\n this.#reservationConfirmDelays.add(1);\n }\n this.#lc.error?.(\n `pg change-log minWatermark ${minWatermark} is later than ` +\n `backupWatermark ${backupWatermark}. Delaying confirmation of ` +\n `snapshot reservation until next backup.`,\n );\n }\n }\n }\n\n #maybeSchedulePGPurge(): void {\n const backupWatermark = this.#backupWatermark;\n if (\n this.#pgPurgeScheduled \|\|\n this.#pgPurgeRunning \|\|\n backupWatermark === undefined \|\|\n this.#pgPurgedWatermark >= backupWatermark\n ) {\n return;\n }\n this.#pgPurgeScheduled = true;\n this.#state.setTimeout(() => {\n this.#pgPurgeScheduled = false;\n this.#pgPurgeRunning = true;\n return this.#purgePGChangeLog().finally(() => {\n this.#pgPurgeRunning = false;\n this.#maybeSchedulePGPurge();\n });\n }, CLEANUP_DELAY_MS);\n }\n\n #maybeScheduleSQLitePurge(): void {\n if (\n this.#sqlitePurgeScheduled \|\|\n this.#sqlitePurgeRunning \|\|\n this.#backupWatermark === undefined \|\|\n this.#sqlitePurgeContinuation === undefined\n ) {\n return;\n }\n const delay =\n this.#sqlitePurgeContinuation === 'immediate' ? 0 : CLEANUP_DELAY_MS;\n this.#sqlitePurgeScheduled = true;\n this.#state.setTimeout(() => {\n this.#sqlitePurgeScheduled = false;\n this.#sqlitePurgeRunning = true;\n return this.#purgeSQLiteChangeLog().finally(() => {\n this.#sqlitePurgeRunning = false;\n this.#maybeScheduleSQLitePurge();\n });\n }, delay);\n }\n\n async #getChangeLogState(): Promise<{\n replicaVersion: string;\n minWatermark: string;\n }> {\n const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n if (!minWatermark) {\n this.#lc.warn?.(\n `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n );\n }\n return {\n replicaVersion: this.#replicaVersion,\n minWatermark: minWatermark ?? this.#replicaVersion,\n };\n }\n\n #getCleanupFloor(): {\n backupWatermark: string;\n purgeWatermark: string;\n current: string[];\n } {\n const backupWatermark = this.#backupWatermark;\n assert(\n backupWatermark !== undefined,\n 'cleanup cannot run without a backup watermark',\n );\n const current = [\n ...this.#forwarder.getAcks(),\n ...(this.#reservations?.getReservedWatermarks() ?? []),\n ];\n // The cleanup delay above is the grace period for disconnected\n // subscribers to reconnect and expose their ACKs. Once it expires, an\n // empty set places no additional constraint on the confirmed backup\n // watermark and must not pin either change log indefinitely.\n return {\n backupWatermark,\n purgeWatermark: min(backupWatermark, ...current),\n current,\n };\n }\n\n async #purgePGChangeLog(): Promise<void> {\n try {\n const {backupWatermark, purgeWatermark, current} =\n this.#getCleanupFloor();\n if (purgeWatermark < backupWatermark) {\n if (this.#loggedBehindWatermark !== purgeWatermark) {\n this.#loggedBehindWatermark = purgeWatermark;\n this.#lc.info?.(\n `At least one client is behind backup ${backupWatermark}`,\n {watermarks: current},\n );\n }\n } else {\n this.#loggedBehindWatermark = undefined;\n }\n if (purgeWatermark <= this.#pgPurgedWatermark) {\n return;\n }\n this.#lc.info?.(`Purging PG changes before ${purgeWatermark} ...`);\n const start = performance.now();\n const deleted = await this.#storer.purgeRecordsBefore(purgeWatermark);\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `Purged ${deleted} PG changes before ${purgeWatermark} (${elapsed} ms)`,\n );\n this.#pgPurgedWatermark = purgeWatermark;\n } catch (e) {\n this.#lc.warn?.(`error purging the PG change log`, e);\n }\n }\n\n async #purgeSQLiteChangeLog(): Promise<void> {\n const scheduler = this.#purgeScheduler;\n if (!scheduler) {\n return;\n }\n try {\n const {backupWatermark, purgeWatermark} = this.#getCleanupFloor();\n // Consume the request before starting. A backup notification that\n // arrives during this pass records another request, which is merged with\n // the continuation returned by this pass rather than being overwritten.\n this.#sqlitePurgeContinuation = undefined;\n if (purgeWatermark < backupWatermark) {\n // Live constraint changes are not edge-triggered. Keep evaluating the\n // floor until it reaches the durable backup watermark, independently\n // of whether the PG implementation still exists.\n this.#requestSQLitePurge('deferred');\n }\n const result = await scheduler.purge(purgeWatermark);\n this.#requestSQLitePurge(result.continuation);\n } catch (e) {\n // purge() is fail-soft, but retain the coordinator's retry if an\n // unexpected caller-boundary failure escapes it.\n this.#requestSQLitePurge('deferred');\n this.#lc.warn?.(`error purging the SQLite change log`, e);\n }\n }\n\n #requestSQLitePurge(continuation: PurgeContinuation \| undefined): void {\n if (!this.#purgeScheduler \|\| continuation === undefined) {\n return;\n }\n if (\n continuation === 'immediate' \|\|\n this.#sqlitePurgeContinuation === undefined\n ) {\n this.#sqlitePurgeContinuation = continuation;\n }\n }\n\n async stop(err?: unknown) {\n this.#state.stop(this.#lc, err);\n this.#stream?.changes.cancel();\n this.#purgeScheduler?.stop();\n this.#comparator?.stop();\n this.#sqliteCatchup?.close();\n this.#changeLogWriter?.close();\n await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);\n }\n\n #recordForwardedTransactionBoundary(\n type: ChangeStreamData[0],\n watermark: string,\n ) {\n switch (type) {\n case 'begin':\n assert(\n this.#currentTransactionCompletion === undefined,\n 'forwarded begin while a transaction is already in progress',\n );\n this.#currentTransactionCompletion =\n resolver<ForwardedTransactionCompletion>();\n this.#transactionForwardStartedAt = performance.now();\n break;\n case 'commit': {\n const completion = this.#currentTransactionCompletion;\n assert(completion, 'forwarded commit without a pending transaction');\n this.#recordTransactionForwardDuration();\n this.#lastForwardedCommitWatermark = watermark;\n this.#currentTransactionCompletion = undefined;\n completion.resolve({kind: 'committed', watermark});\n break;\n }\n case 'rollback': {\n const completion = this.#currentTransactionCompletion;\n assert(completion, 'forwarded rollback without a pending transaction');\n const committed = this.#lastForwardedCommitWatermark;\n assert(committed, 'last forwarded commit watermark is not initialized');\n this.#transactionForwardStartedAt = undefined;\n this.#currentTransactionCompletion = undefined;\n completion.resolve({kind: 'rolled-back', watermark: committed});\n break;\n }\n }\n }\n\n #recordTransactionForwardDuration() {\n const startedAt = this.#transactionForwardStartedAt;\n this.#transactionForwardStartedAt = undefined;\n if (startedAt !== undefined) {\n this.#transactionForwardDuration.recordMs(performance.now() - startedAt, {\n sqlite_change_log: this.#changeLogWriter?.enabled ? 'on' : 'off',\n });\n }\n }\n\n /\n Closes and forgets the cached catchup coordinator, so that a later\n * subscription re-opens the log from scratch rather than reading a handle\n * whose file is gone.\n /\n #closeSQLiteCatchup() {\n this.#sqliteCatchup?.close();\n this.#sqliteCatchup = undefined;\n }\n\n #captureRequiredHead(): string \| Promise<string> {\n const committed = this.#lastForwardedCommitWatermark;\n assert(committed, 'last forwarded commit watermark is not initialized');\n return (\n this.#currentTransactionCompletion?.promise.then(\n completion => completion.watermark,\n ) ?? committed\n );\n }\n\n #selectSQLiteCatchup(\n lc: LogContext,\n ctx: SubscriberContext,\n ): SQLiteCatchupSelection \| undefined {\n const opts = this.#sqliteCatchupOptions;\n if (this.#lastForwardedCommitWatermark === undefined \|\| !opts) {\n this.#recordCatchupRoute('pg', 'not-ready');\n return undefined;\n }\n // SQLite catchup is only for disposable serving replicas. Backup\n // subscribers retain the existing PG recovery policy until RMv2 can\n // resume the canonical replica directly from replica/backup/slot state.\n // Enforce this before the selector so no canary policy can override it.\n if (ctx.mode !== 'serving') {\n lc.info?.(`not serving backup subscriber ${ctx.id} from SQLite catchup`);\n this.#recordCatchupRoute('pg', 'ineligible-mode');\n return undefined;\n }\n\n let route: ChangeLogReadRoute \| undefined;\n if (this.#readRouter) {\n route = this.#readRouter.consume(ctx.taskID);\n if (route.source === 'pg') {\n if (route.reason === 'cold-log') {\n lc.info?.(\n `serving ${ctx.id} from PG catchup: SQLite change log is still warming`,\n {sqliteChangeLogCoverage: route.coverage},\n );\n } else if (route.reason === 'log-unavailable') {\n this.#declineSQLiteCatchup(\n lc,\n ctx,\n opts,\n `${opts.changeLogFile} is unavailable or incompatible`,\n );\n } else {\n lc.debug?.(\n `serving ${ctx.id} from PG catchup: SQLite route ${route.reason}`,\n ...(route.coverage\n ? [{sqliteChangeLogCoverage: route.coverage}]\n : []),\n );\n }\n this.#recordCatchupRoute('pg', route.reason);\n return undefined;\n }\n const coverage = route.coverage;\n assert(coverage, 'a SQLite route must carry its covered range');\n if (ctx.watermark < coverage.minWatermark) {\n lc.info?.(\n `serving ${ctx.id} from PG catchup: subscriber watermark ` +\n `${ctx.watermark} is below the SQLite change-log minimum ` +\n coverage.minWatermark,\n {sqliteChangeLogCoverage: coverage},\n );\n this.#recordCatchupRoute('pg', 'watermark-uncovered');\n return undefined;\n }\n }\n\n // `shouldUse` remains as a test hook and an optional extra policy gate.\n // Production selection comes from #readRouter.\n if (!this.#readRouter && !opts.shouldUse?.(ctx)) {\n this.#recordCatchupRoute('pg', 'selector');\n return undefined;\n }\n if (this.#readRouter && opts.shouldUse && !opts.shouldUse(ctx)) {\n this.#recordCatchupRoute('pg', 'selector');\n return undefined;\n }\n\n const catchup =\n this.#sqliteCatchup ?? this.#openSQLiteCatchup(lc, opts, ctx);\n if (catchup) {\n return {\n catchup,\n reason: route?.reason ?? 'selector',\n coverage: route?.coverage,\n // Legacy focused-test selection has no warm classification.\n logWarm:\n route === undefined ? undefined : route.reason !== 'selected-cold',\n };\n } else {\n this.#recordCatchupRoute('pg', 'log-unavailable');\n }\n return undefined;\n }\n\n #recordCatchupRoute(source: 'pg' \| 'sqlite', reason: string): void {\n this.#catchupRoutes.add(1, {source, reason});\n }\n\n /\n Opens the change log and, if it can serve, wraps it in the coordinator that\n * subsequent subscriptions reuse.\n \n The log may not exist yet -- the writer creates it at the stream loop's\n * first reconcile, and deletes it when it fails soft -- or may exist without\n * content, or may be unreadable. None of those can be allowed to fail a\n * subscription:\n * this is the last point at which PG catchup is still available -- past\n * `Forwarder.add()` the subscriber is committed to SQLite -- so each declines\n * here instead. Neither the failure nor the reader is retained, so a later\n * subscription retries from scratch.\n /\n #openSQLiteCatchup(\n lc: LogContext,\n opts: SQLiteCatchupOptions,\n ctx: SubscriberContext,\n ): SQLiteChangeLogCatchup \| undefined {\n let reader: SQLiteChangeLogReader \| undefined;\n try {\n reader = new SQLiteChangeLogReader(this.#lc, opts.changeLogFile);\n // `plan()` doubles as the readiness check: it reports `not-ready` when\n // the writer has not created or seeded the stream table. The barrier\n // cannot wait its way out of that, since a log with no head can never\n // reach the required one.\n if (reader.plan(ctx.watermark).kind === 'not-ready') {\n reader.close();\n this.#declineSQLiteCatchup(\n lc,\n ctx,\n opts,\n `${opts.changeLogFile} has no changes to serve yet`,\n );\n return undefined;\n }\n } catch (e) {\n // An absent file is the common case, since a readonly handle cannot\n // create one. A corrupt or truncated file lands here too, and is equally\n // not a reason to fail the subscription.\n reader?.close();\n this.#declineSQLiteCatchup(\n lc,\n ctx,\n opts,\n `cannot read ${opts.changeLogFile}`,\n e,\n );\n return undefined;\n }\n this.#changeLogUnavailableSince = undefined;\n // Readiness is checked once, on the way to the cached coordinator: a log\n // with content keeps it. Purging preserves the latest transaction as a\n // catchup boundary, and `reconcileChangeLog` reseeds inside a single\n // transaction, so a reader never observes an emptied log.\n this.#sqliteCatchup = new SQLiteChangeLogCatchup(\n this.#lc,\n this.#forwarder,\n reader,\n {\n batchSize: opts.readBatchRows,\n barrierTimeoutMs: opts.barrierTimeoutMs,\n barrierPollIntervalMs: opts.barrierPollIntervalMs,\n cleanupGuard: opts.cleanupGuard,\n onFailure: failure => {\n this.#lc.warn?.(\n `temporarily disabling SQLite catchup after ${failure}`,\n );\n this.#readRouter?.trip();\n },\n },\n );\n return this.#sqliteCatchup;\n }\n\n /\n Reports falling back to PG catchup, tracking how long the log has been\n * unavailable so that a subscription that merely arrived before the writer's\n * first reconcile does not look like an incident.\n /\n #declineSQLiteCatchup(\n lc: LogContext,\n ctx: SubscriberContext,\n opts: SQLiteCatchupOptions,\n reason: string,\n error?: unknown,\n ): void {\n const now = Date.now();\n this.#changeLogUnavailableSince ??= now;\n const unavailableMs = now - this.#changeLogUnavailableSince;\n const threshold =\n opts.notReadyWarnThresholdMs ??\n DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS;\n lc[unavailableMs >= threshold ? 'warn' : 'debug']?.(\n `serving ${ctx.id} from PG catchup: ${reason} ` +\n `(unavailable for ${unavailableMs} ms)`,\n ...(error === undefined ? [] : [error]),\n );\n }\n}\n\ntype ForwardedTransactionCompletion =\n \| {kind: 'committed'; watermark: string}\n \| {kind: 'rolled-back'; watermark: string};\n\ntype SQLiteCatchupSelection = {\n readonly catchup: SQLiteChangeLogCatchup;\n readonly reason: string;\n readonly coverage: SQLiteChangeLogCoverage \| undefined;\n /\n Whether the router classified the log as warm for this subscriber, or\n * undefined when selection did not run through the router (focused tests).\n /\n readonly logWarm: boolean \| undefined;\n};\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n// `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS 3;\n\nconst ROLLBACK_JSON = JSON.stringify([\n 'rollback',\n {tag: 'rollback'},\n] satisfies Rollback);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4LA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,cACA,WACA,WACA,MACA,eAAe,YACiB;CAChC,MAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,aAAa,KAAA,GACb,YACF;CAOA,MAAM,oBACJ,iBACA,QAEA,SACE,KACE;EAAC;EAAK,YAAY,GAAE,qBAAqB,gBAAe;CAAC,GAKzD,SAAS,OACX,CACF;CAEF,MAAM,EAAC,mBAAkB;CACzB,OAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,kBACA,gBACA,cACA,4BACA,cACA,WACA,WACA,MACA,YACF;AACF;AAEA,IAAM,8CAA8C;AAQpD,IAAM,mDAAmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4IzD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CAUA,WAAoB,SAAS;CAE7B,aAAsB,mBACpB,eACA,gBACA,kCACF;CACA,iBAA0B,mBACxB,eACA,WACA,qDACF;CAIA,8BAAuC,4BACrC,eACA,gCACA,yIAEF;CACA,iBAA0B,mBACxB,eACA,oCACA,sEACF;CACA,wBAAiC,mBAC/B,eACA,2CACA,qPAIF;CACA,4BAAqC,mBACnC,eACA,gDACA,2LAGF;CAEA;CACA,+BAA+B;CAC/B;CACA,qBAA6B;;;;;;;CAO7B;CACA;CACA;CAGA,oBAAoB;CACpB,kBAAkB;CAClB,wBAAwB;CACxB,sBAAsB;CACtB;CACA;CACA;CACA;CACA;CACA;CAIA,YACE,IACA,OACA,QACA,kBACA,mBACA,kBACA,gBACA,QACA,4BACA,cACA,kBACA,WACA,MACA,eAAe,YACf;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EACxD,KAAKC,SAAS;EACd,KAAKC,oBAAoB;EACzB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,UAAU,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,kBACA,iBACA,aAAY,KAAKQ,OAAO,iBAAiB,SAAS,GAAG,SAAS,IAC9D,QAAO,KAAK,KAAK,GAAG,GACpB,IACF;EACA,KAAKP,aAAa,IAAI,UAAU,IAAI;GAClC,uCACE,KAAK;GACP,wCACE,KAAK;EACT,CAAC;EACD,MAAM,eAAe,KAAK;EAC1B,MAAM,gBAAgB,KAAK;EAC3B,MAAM,iBAAiB,KAAK;EAC5B,KAAKS,cACH,gBAAgB,iBAAiB,iBAC7B,IAAI,0BAA0B;GAC5B;GACA,aAAa,aAAa;GAC1B,iBAAiB,aAAa;GAC9B,aAAa,aAAa;GAC1B,mBAAmB,aAAa;GAChC,KAAK,aAAa;GAClB,SACE,aAAa,kBAEX,uBACE,IACA,eAAe,eACf,cAAc,QAChB;EACN,CAAC,IACD,KAAA;EACN,KAAKR,gBAAgB,eACjB,IAAI,qBAAqB,IAAI,eAAc,WAAU;GACnD,KAAKQ,aAAa,QAAQ,MAAM;GAChC,KAAKJ,iBAAiB,OAAO,MAAM;EACrC,CAAC,IACD,KAAA;EACJ,KAAKH,8BAA8B;EACnC,KAAKE,mBAAmB,KAAK,wBACzB,IAAI,sBAAsB,IAAI;GAC5B,GAAG,KAAK;GACR,WAAU,cAAa;IACrB,KAAKe,gBAAgB,kBAAkB,SAAS;IAGhD,KAAKd,iBAAiB,aAAa;GACrC;GAIA,kBAAkB;IAIhB,KAAKI,aAAa,KAAK,IAAI;IAC3B,KAAKW,oBAAoB;IACzB,KAAKd,aAAa,KAAK;GACzB;GACA,iBAAiB;IACf,KAAKc,oBAAoB;IAEzB,KAAKd,aAAa,WAAW;GAC/B;EACF,CAAC,IACD,KAAA;EAMJ,KAAKD,kBACH,KAAK,wBAAwB,KAAKD,mBAC9B,IAAI,8BACF,UACM,KAAKA,kBAAkB,kBACvB,KAAKJ,WAAW,QAAQ,GAC9B,KAAK,oBACP,IACA,KAAA;EACN,KAAKO,SAAS,IAAI,cAAc;GAC9B,kBAAkB;GAClB,aAAa,cAAc,sBAAsB;EACnD,CAAC;EACD,MAAM,gBAAgB,KAAK,yBACvB,4BAA4B,IAAI,KAAK,uBAAuB,WAAW,IACvE,KAAA;EACJ,KAAKC,eAAe,IAAI,qBACtB,IACA;GACE,qBAAqB;GACrB,iBAAiB,kBAAkB,KAAA;EACrC,GACA;GACE,mBACE,KAAKT,QAAQ,uCAAuC;GAGtD,eAAe,KAAK,aAAa,EAAE;GAGnC,qBAAqB,YAAY,SAC/B,aACI,KAAKK,kBAAkB,UAAU,UAAU,IAC3C,KAAKA,kBAAkB,iBAAiB,IAAI;GAClD,iBAAiB,KAAKA,kBAAkB;EAC1C,CACF;EACA,KAAKD,wBAAwB,KAAK,gBAC9B;GACE,GAAG,KAAK;GAGR,cACE,KAAK,cAAc,gBACnB,KAAKE,iBAAiB;EAC1B,IACA,KAAA;EAEJ,KAAKC,cACH,KAAK,0BACL,KAAK,yBACL,KAAK,gBACD,IAAI,0BACF,IACA,OACA,KAAK,cAAc,eACnB,KAAK,sBAAsB,UAC3B,KAAKP,SACL;GAAC;GAAc,GAAG,KAAK;EAAsB,CAC/C,IACA,KAAA;EACN,KAAKsB,aAAa;EAClB,KAAKX,aAAa;EAClB,KAAKC,SAAS,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,YAAY;EAC/D,KAAKW,gBAAgB,EAAC,KAAK,SAAQ;CACrC;CAEA,MAAM,MAAM;EACV,KAAK5B,IAAI,OAAO,wBAAwB;EAExC,KAAKM,WAAW,qBAAqB;EAErC,MAAM,gBAAgB,MAAM,KAAKF,QAAQ,iBAAiB;EAC1D,IAAI,eAAe;GACjB,KAAKwB,cAAc,YAAY,EAC7B,gBAAgB,cAAc,eAChC;GAKA,KAAKC,+BAA+B,cAAc;EACpD;EAIA,MAAM,KAAKxB,QAAQ,gBAAgB,KAAKsB,UAAU;EAClD,KAAKA,aAAa;EAIlB,MAAM,sBAAsB,wBAAwB,KAAK;EAEzD,OAAO,KAAKV,OAAO,UAAU,GAAG;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;GACrB,IAAI;IAIF,MAAM,EAAC,eAAe,qBACpB,MAAM,KAAKH,aAAa,WAAW;IAIrC,KAAKgB,gCAAgC;IACrC,MAAM,SAAS,MAAM,KAAK1B,QAAQ,YAChC,eACA,gBACF;IACA,KAAKC,QAAQ,IAAI,EAAE,OAAM,MAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;IAEtD,KAAK0B,UAAU;IACf,IACE,KAAKd,OAAO,aAAa,IACzB,6CAIA,KAAKT,4BAA4B,QAC/B,KAAKR,KACL,eACA,oBAAoB,eACtB;IAEF,YAAY;IAEZ,KAAKa,OAAO,MAAM,OAAO,IAAI;IAC7B,WAAW,MAAM,UAAU,OAAO,SAAS;KACzC,KAAKA,OAAO,gBAAgB,MAAM;KAElC,MAAM,CAAC,MAAM,OAAO;KACpB,QAAQ,MAAR;MACE,KAAK;OACH,IACE,IAAI,aACJ,IAAI,UAAU,YAAY,gBACxB,KAAKgB,8BACP;QAKA,KAAKD,cAAc,YAAY,IAAI;QACnC,KAAKC,+BACH,IAAI,UAAU,YAAY;QAC5B,KAAKvB,WAAW,WAAW,KAAKsB,aAAa;OAC/C;OACA;MACF,KAAK;OACH,MAAM,KAAKI,sBAAsB,GAAG;OACpC;MACF,KAAK;OACH,YAAY,OAAO,GAAG;OACtB;MACF,KAAK;OACH,IAAI,cAAc,OAAO,GAAG,WAC1B,MAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,WAC9E;OAEF,KAAKb,WAAW,IAAI,CAAC;OACrB;MACF;OACE,IAAI,SAAS,QACX,KAAKC,eAAe,IAAI,CAAC;OAE3B,IAAI,cAAc,MAChB,MAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,kCAC7B;OAEF;KACJ;KAEA,MAAM,OAAO,KAAKf,QAAQ,MAAM,WAAW,MAAM;KAWjD,KAAKK,kBAAkB,MAAM,QAAQ,IAAI;KACzC,MAAM,QAA2B;MAAC;MAAW,OAAO,GAAG;MAAK;KAAI;KAChE,kBAAkB,KAAK;KACvB,IAAI,iBAAiB,qBAAqB;MAExC,KAAKJ,WAAW,QAAQ,KAAK;MAC7B,KAAK2B,oCAAoC,MAAM,MAAM,EAAE;KACzD,OAAO;MAQL,MAAM,YAAY,KAAK3B,WAAW,uBAAuB,KAAK;MAK9D,KAAK2B,oCAAoC,MAAM,MAAM,EAAE;MACvD,MAAM,eACJ,WACA,OAAO,QAAQ,QACf,KAAKhB,OAAO,MACd;MACA,iBAAiB;KACnB;KAEA,IAAI,SAAS,YAAY,SAAS,YAChC,YAAY;KAId,MAAM,eAAe,KAAKZ,QAAQ,aAAa;KAC/C,IAAI,cACF,MAAM,eACJ,cACA,OAAO,QAAQ,QACf,KAAKY,OAAO,MACd;IAEJ;GACF,SAAS,GAAG;IACV,MAAM;GACR,UAAU;IACR,KAAKc,SAAS,QAAQ,OAAO;IAC7B,KAAKA,UAAU,KAAA;GACjB;GAGA,IAAI,WAAW;IACb,KAAK/B,IAAI,OAAO,oCAAoC,WAAW;IAC/D,KAAKK,QAAQ,MAAM;IAInB,KAAKK,kBAAkB,MAAM;IAG7B,KAAKC,iBAAiB,aAAa;IACnC,KAAKL,WAAW,QAAQ;KAAC;KAAW;KAAY;IAAa,CAAC;IAC9D,KAAK2B,oCAAoC,YAAY,SAAS;GAChE;GAGA,MAAM,QAAQ,IAAI;IAChB,KAAK5B,QAAQ,KAAK;IAClB,KAAKY,OAAO,QAAQ,KAAKjB,KAAK,GAAG;IACjC,KAAKiB,OAAO,aAAa,8CACrB,qBACE,KAAKjB,KACL,uBAAuB,KAAKA,KAAK,eAAe,GAAG,CACrD,IACA;GACN,CAAC;EACH;EAEA,KAAKM,WAAW,oBAAoB;EACpC,KAAKN,IAAI,OAAO,wBAAwB;CAC1C;CAEA,MAAMgC,sBAAsB,KAA6B;EACvD,KAAKhC,IAAI,OAAO,4BAA4B,GAAG;EAC/C,MAAM,EAAC,QAAO;EAEd,QAAQ,KAAR;GACE,KAAK;IACH,MAAM,kBACJ,KAAKE,kBAAkB,yBAAyB,CAAC,GACjD,KAAKD,MACP;IACA,MAAM,wBACJ,KAAKD,KACL,eACA,IAAI,WAAW,mBACf,IAAI,YACN;IACA,IAAI,KAAKgB,YAAY;KACnB,KAAKhB,IAAI,OAAO,8BAA8B;KAC9C,MAAM,KAAK,KAAK,IAAI,gBAAgB,CAAC;IACvC;IACA;GACF,SACE,YAAY,GAAG;EACnB;CACF;CAEA,MAAM,UAAU,KAAiD;EAC/D,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;EAC/D,IAAI,SAAS,WACX,KAAKkB,SAAS,QAAQ;EAExB,IAAI,0BAA0B,CAAC;EAC/B,MAAM,aAAa,aAAa,OAAe,EAC7C,eAAe,kBAAkB,EACnC,CAAC;EAID,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,MACA,WACA,kBACM,KAAKU,eACX,CAAC,CACH;EACA,MAAM,KAAK,KAAK5B,IAAI,YAAY,cAAc,WAAW,EAAE;EAC3D,MAAM,4BAA4B;GAChC,GAAG,OAAO,uBAAuB,WAAW,IAAI;GAChD,KAAKM,WAAW,OAAO,UAAU;EACnC;EACA,oBAAoB;EACpB,IAAI,mBAAmB,KAAKH,iBAAiB;GAC3C,GAAG,OAAO,2CAA2C,gBAAgB;GACrE,WAAW,MACT,GACA,8BACE,KAAKA,gBACN,cAAc,eAAe,EAChC;EACF,OAAO;GACL,GAAG,OAAO,qBAAqB,WAAW,IAAI;GAE9C,MAAM,sBAAsB;IAG1B,oBAAoB;IACpB,KAAKG,WAAW,IAAI,UAAU;IAC9B,KAAKD,QAAQ,QAAQ,YAAY,IAAI;GACvC;GACA,MAAM,kBAAkB,KAAK6B,qBAAqB,IAAI,GAAG;GACzD,IAAI,CAAC,iBACH,cAAc;QACT;IACL,MAAM,EAAC,SAAS,QAAQ,UAAU,YAAW;IAC7C,0BAA0B,QAAQ,OAAO,UAAU;IACnD,MAAM,eAAe,MAAM,QAAQ,QACjC,kBACM,KAAKC,qBAAqB,GAChC,EAAC,QAAO,CACV;IACA,QAAQ,aAAa,MAArB;KACE,KAAK;MACH,GAAG,QACD,WAAW,IAAI,GAAG,uBAClB,GAAI,WAAW,CAAC,EAAC,yBAAyB,SAAQ,CAAC,IAAI,CAAC,CAC1D;MACA,KAAKC,oBAAoB,UAAU,MAAM;MACzC;KACF,KAAK;MACH,GAAG,OACD,WAAW,IAAI,GAAG,yCACb,IAAI,UAAU,4CACjB,aAAa,YACjB;MACA,KAAKA,oBAAoB,MAAM,qBAAqB;MACpD,cAAc;MACd;KACF,KAAK;MAOH,GAAG,QACD,WAAW,IAAI,GAAG,uDAElB,aAAa,KACf;MACA,KAAKA,oBAAoB,MAAM,qBAAqB;MACpD,cAAc;MACd;KACF,KAAK;MAIH,KAAKA,oBAAoB,UAAU,sBAAsB;MACzD;KACF,SACE,YAAY,YAAY;IAC5B;GACF;EACF;EAGA,KAAK7B,eAAe,MAAM,IAAI,MAAM;EACpC,OAAO;CACT;CAEA,MAAM,yBACJ,QACkC;EAClC,IAAI,CAAC,KAAKA,eACR,MAAM,IAAI,MAAM,4BAA4B;EAE9C,MAAM,aAAa,KAAKA,cAAc,KAAK,MAAM;EAEjD,IAAI;GAGF,OAAO,KAAKI,iBAAiB,MAAM,MAAM,KAAK;GAI9C,IAAI,CAAC,KAAKJ,cAAc,UAAU,QAAQ,UAAU,GAClD,OAAO;GAKT,KAAKQ,aAAa,IAAI,MAAM;GAE5B,MAAM,KAAKsB,qBAAqB;GAChC,OAAO;EACT,SAAS,GAAG;GAOV,WAAW,OAAO;GAClB,MAAM;EACR;CACF;CAEA,qBAAqB,WAAmB;EACtC,KAAKC,mBAAmB;EACxB,KAAKzB,OAAO,YAAY,SAAS;EAIjC,KAAK0B,oBAAoB,UAAU;EACnC,KAAKC,sBAAsB;EAC3B,KAAKC,0BAA0B;EAK/B,KAAUJ,qBAAqB,EAAE,OAAM,MACrC,KAAKrC,IAAI,OAAO,yCAAyC,CAAC,CAC5D;CACF;CAEA,MAAMqC,uBAAuB;EAC3B,MAAM,kBAAkB,KAAKC;EAC7B,MAAM,eAAe,KAAK/B;EAC1B,IACE,oBAAoB,KAAA,KACpB,CAAC,cAAc,sBAAsB,GAErC;EAUF,MAAM,UAAU,MAAM,KAAKmC,mBAAmB;EAC9C,KAAK,MAAM,UAAU,aAAa,mBAAmB,GAAG;GACtD,IAAI,QAAQ,KAAK3B,aAAa,KAAK,MAAM;GAGzC,IAAI,KAAKA,eAAe,UAAU,KAAA,GAChC;GAGF,IAAI,OAAO,WAAW,UAAU;IAC9B,MAAM,WAAW,KACf,MAAM,UACN,oDACF;IACA,IAAI,SAAS,eAAe,iBAAiB;KAI3C,KAAKf,IAAI,OACP,YAAY,OAAO,4CACd,SAAS,aAAa,mCACzB,eACJ;KACA,KAAKuB,sBAAsB,IAAI,CAAC;KAChC,QAAQ,KAAK,KAAKR,WAAW,EAAE,OAAO,MAAM;IAC9C;GACF;GAEA,MAAM,SAAS,OAAO,UAAU;GAChC,IAAI;GACJ,IAAI,OAAO,WAAW,UAGpB,eAAe,KACb,MAAM,UACN,oDACF,EAAE;QAEF,CAAC,CAAC,gBAAgB;GAGpB,IAAI,gBAAgB,iBAClB,aAAa,WACX,QACA,QAAQ,gBACR,iBACA,MACF;QACK;IAIL,IAAI,aAAa,wBAAwB,MAAM,GAC7C,KAAKS,0BAA0B,IAAI,CAAC;IAEtC,KAAKxB,IAAI,QACP,8BAA8B,aAAa,iCACtB,gBAAgB,mEAEvC;GACF;EACF;CACF;CAEA,wBAA8B;EAC5B,MAAM,kBAAkB,KAAKsC;EAC7B,IACE,KAAKK,qBACL,KAAKC,mBACL,oBAAoB,KAAA,KACpB,KAAKC,sBAAsB,iBAE3B;EAEF,KAAKF,oBAAoB;EACzB,KAAK1B,OAAO,iBAAiB;GAC3B,KAAK0B,oBAAoB;GACzB,KAAKC,kBAAkB;GACvB,OAAO,KAAKE,kBAAkB,EAAE,cAAc;IAC5C,KAAKF,kBAAkB;IACvB,KAAKJ,sBAAsB;GAC7B,CAAC;EACH,GAAG,gBAAgB;CACrB;CAEA,4BAAkC;EAChC,IACE,KAAKO,yBACL,KAAKC,uBACL,KAAKV,qBAAqB,KAAA,KAC1B,KAAKW,6BAA6B,KAAA,GAElC;EAEF,MAAM,QACJ,KAAKA,6BAA6B,cAAc,IAAI;EACtD,KAAKF,wBAAwB;EAC7B,KAAK9B,OAAO,iBAAiB;GAC3B,KAAK8B,wBAAwB;GAC7B,KAAKC,sBAAsB;GAC3B,OAAO,KAAKE,sBAAsB,EAAE,cAAc;IAChD,KAAKF,sBAAsB;IAC3B,KAAKP,0BAA0B;GACjC,CAAC;EACH,GAAG,KAAK;CACV;CAEA,MAAMC,qBAGH;EACD,MAAM,eAAe,MAAM,KAAKrC,QAAQ,0BAA0B;EAClE,IAAI,CAAC,cACH,KAAKL,IAAI,OACP,8EACF;EAEF,OAAO;GACL,gBAAgB,KAAKG;GACrB,cAAc,gBAAgB,KAAKA;EACrC;CACF;CAEA,mBAIE;EACA,MAAM,kBAAkB,KAAKmC;EAC7B,OACE,oBAAoB,KAAA,GACpB,+CACF;EACA,MAAM,UAAU,CACd,GAAG,KAAKhC,WAAW,QAAQ,GAC3B,GAAI,KAAKC,eAAe,sBAAsB,KAAK,CAAC,CACtD;EAKA,OAAO;GACL;GACA,gBAAgB,IAAI,iBAAiB,GAAG,OAAO;GAC/C;EACF;CACF;CAEA,MAAMuC,oBAAmC;EACvC,IAAI;GACF,MAAM,EAAC,iBAAiB,gBAAgB,YACtC,KAAKK,iBAAiB;GACxB,IAAI,iBAAiB;QACf,KAAKC,2BAA2B,gBAAgB;KAClD,KAAKA,yBAAyB;KAC9B,KAAKpD,IAAI,OACP,wCAAwC,mBACxC,EAAC,YAAY,QAAO,CACtB;IACF;UAEA,KAAKoD,yBAAyB,KAAA;GAEhC,IAAI,kBAAkB,KAAKP,oBACzB;GAEF,KAAK7C,IAAI,OAAO,6BAA6B,eAAe,KAAK;GACjE,MAAM,QAAQ,YAAY,IAAI;GAC9B,MAAM,UAAU,MAAM,KAAKK,QAAQ,mBAAmB,cAAc;GACpE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;GACrD,KAAKL,IAAI,OACP,UAAU,QAAQ,qBAAqB,eAAe,IAAI,QAAQ,KACpE;GACA,KAAK6C,qBAAqB;EAC5B,SAAS,GAAG;GACV,KAAK7C,IAAI,OAAO,mCAAmC,CAAC;EACtD;CACF;CAEA,MAAMkD,wBAAuC;EAC3C,MAAM,YAAY,KAAKvC;EACvB,IAAI,CAAC,WACH;EAEF,IAAI;GACF,MAAM,EAAC,iBAAiB,mBAAkB,KAAKwC,iBAAiB;GAIhE,KAAKF,2BAA2B,KAAA;GAChC,IAAI,iBAAiB,iBAInB,KAAKV,oBAAoB,UAAU;GAErC,MAAM,SAAS,MAAM,UAAU,MAAM,cAAc;GACnD,KAAKA,oBAAoB,OAAO,YAAY;EAC9C,SAAS,GAAG;GAGV,KAAKA,oBAAoB,UAAU;GACnC,KAAKvC,IAAI,OAAO,uCAAuC,CAAC;EAC1D;CACF;CAEA,oBAAoB,cAAmD;EACrE,IAAI,CAAC,KAAKW,mBAAmB,iBAAiB,KAAA,GAC5C;EAEF,IACE,iBAAiB,eACjB,KAAKsC,6BAA6B,KAAA,GAElC,KAAKA,2BAA2B;CAEpC;CAEA,MAAM,KAAK,KAAe;EACxB,KAAKhC,OAAO,KAAK,KAAKjB,KAAK,GAAG;EAC9B,KAAK+B,SAAS,QAAQ,OAAO;EAC7B,KAAKpB,iBAAiB,KAAK;EAC3B,KAAKC,aAAa,KAAK;EACvB,KAAKa,gBAAgB,MAAM;EAC3B,KAAKf,kBAAkB,MAAM;EAC7B,MAAM,QAAQ,WAAW,CAAC,KAAKL,QAAQ,KAAK,GAAG,KAAKD,QAAQ,KAAK,CAAC,CAAC;CACrE;CAEA,oCACE,MACA,WACA;EACA,QAAQ,MAAR;GACE,KAAK;IACH,OACE,KAAKiD,kCAAkC,KAAA,GACvC,4DACF;IACA,KAAKA,gCACH,SAAyC;IAC3C,KAAKC,+BAA+B,YAAY,IAAI;IACpD;GACF,KAAK,UAAU;IACb,MAAM,aAAa,KAAKD;IACxB,OAAO,YAAY,gDAAgD;IACnE,KAAKE,kCAAkC;IACvC,KAAKzB,gCAAgC;IACrC,KAAKuB,gCAAgC,KAAA;IACrC,WAAW,QAAQ;KAAC,MAAM;KAAa;IAAS,CAAC;IACjD;GACF;GACA,KAAK,YAAY;IACf,MAAM,aAAa,KAAKA;IACxB,OAAO,YAAY,kDAAkD;IACrE,MAAM,YAAY,KAAKvB;IACvB,OAAO,WAAW,oDAAoD;IACtE,KAAKwB,+BAA+B,KAAA;IACpC,KAAKD,gCAAgC,KAAA;IACrC,WAAW,QAAQ;KAAC,MAAM;KAAe,WAAW;IAAS,CAAC;IAC9D;GACF;EACF;CACF;CAEA,oCAAoC;EAClC,MAAM,YAAY,KAAKC;EACvB,KAAKA,+BAA+B,KAAA;EACpC,IAAI,cAAc,KAAA,GAChB,KAAKjC,4BAA4B,SAAS,YAAY,IAAI,IAAI,WAAW,EACvE,mBAAmB,KAAKX,kBAAkB,UAAU,OAAO,MAC7D,CAAC;CAEL;;;;;;CAOA,sBAAsB;EACpB,KAAKe,gBAAgB,MAAM;EAC3B,KAAKA,iBAAiB,KAAA;CACxB;CAEA,uBAAiD;EAC/C,MAAM,YAAY,KAAKK;EACvB,OAAO,WAAW,oDAAoD;EACtE,OACE,KAAKuB,+BAA+B,QAAQ,MAC1C,eAAc,WAAW,SAC3B,KAAK;CAET;CAEA,qBACE,IACA,KACoC;EACpC,MAAM,OAAO,KAAK5C;EAClB,IAAI,KAAKqB,kCAAkC,KAAA,KAAa,CAAC,MAAM;GAC7D,KAAKM,oBAAoB,MAAM,WAAW;GAC1C;EACF;EAKA,IAAI,IAAI,SAAS,WAAW;GAC1B,GAAG,OAAO,iCAAiC,IAAI,GAAG,qBAAqB;GACvE,KAAKA,oBAAoB,MAAM,iBAAiB;GAChD;EACF;EAEA,IAAI;EACJ,IAAI,KAAKrB,aAAa;GACpB,QAAQ,KAAKA,YAAY,QAAQ,IAAI,MAAM;GAC3C,IAAI,MAAM,WAAW,MAAM;IACzB,IAAI,MAAM,WAAW,YACnB,GAAG,OACD,WAAW,IAAI,GAAG,uDAClB,EAAC,yBAAyB,MAAM,SAAQ,CAC1C;SACK,IAAI,MAAM,WAAW,mBAC1B,KAAKyC,sBACH,IACA,KACA,MACA,GAAG,KAAK,cAAc,gCACxB;SAEA,GAAG,QACD,WAAW,IAAI,GAAG,iCAAiC,MAAM,UACzD,GAAI,MAAM,WACN,CAAC,EAAC,yBAAyB,MAAM,SAAQ,CAAC,IAC1C,CAAC,CACP;IAEF,KAAKpB,oBAAoB,MAAM,MAAM,MAAM;IAC3C;GACF;GACA,MAAM,WAAW,MAAM;GACvB,OAAO,UAAU,6CAA6C;GAC9D,IAAI,IAAI,YAAY,SAAS,cAAc;IACzC,GAAG,OACD,WAAW,IAAI,GAAG,yCACb,IAAI,UAAU,4CACjB,SAAS,cACX,EAAC,yBAAyB,SAAQ,CACpC;IACA,KAAKA,oBAAoB,MAAM,qBAAqB;IACpD;GACF;EACF;EAIA,IAAI,CAAC,KAAKrB,eAAe,CAAC,KAAK,YAAY,GAAG,GAAG;GAC/C,KAAKqB,oBAAoB,MAAM,UAAU;GACzC;EACF;EACA,IAAI,KAAKrB,eAAe,KAAK,aAAa,CAAC,KAAK,UAAU,GAAG,GAAG;GAC9D,KAAKqB,oBAAoB,MAAM,UAAU;GACzC;EACF;EAEA,MAAM,UACJ,KAAKX,kBAAkB,KAAKgC,mBAAmB,IAAI,MAAM,GAAG;EAC9D,IAAI,SACF,OAAO;GACL;GACA,QAAQ,OAAO,UAAU;GACzB,UAAU,OAAO;GAEjB,SACE,UAAU,KAAA,IAAY,KAAA,IAAY,MAAM,WAAW;EACvD;OAEA,KAAKrB,oBAAoB,MAAM,iBAAiB;CAGpD;CAEA,oBAAoB,QAAyB,QAAsB;EACjE,KAAKd,eAAe,IAAI,GAAG;GAAC;GAAQ;EAAM,CAAC;CAC7C;;;;;;;;;;;;;;CAeA,mBACE,IACA,MACA,KACoC;EACpC,IAAI;EACJ,IAAI;GACF,SAAS,IAAI,sBAAsB,KAAKtB,KAAK,KAAK,aAAa;GAK/D,IAAI,OAAO,KAAK,IAAI,SAAS,EAAE,SAAS,aAAa;IACnD,OAAO,MAAM;IACb,KAAKwD,sBACH,IACA,KACA,MACA,GAAG,KAAK,cAAc,6BACxB;IACA;GACF;EACF,SAAS,GAAG;GAIV,QAAQ,MAAM;GACd,KAAKA,sBACH,IACA,KACA,MACA,eAAe,KAAK,iBACpB,CACF;GACA;EACF;EACA,KAAKE,6BAA6B,KAAA;EAKlC,KAAKjC,iBAAiB,IAAI,uBACxB,KAAKzB,KACL,KAAKM,YACL,QACA;GACE,WAAW,KAAK;GAChB,kBAAkB,KAAK;GACvB,uBAAuB,KAAK;GAC5B,cAAc,KAAK;GACnB,YAAW,YAAW;IACpB,KAAKN,IAAI,OACP,8CAA8C,SAChD;IACA,KAAKe,aAAa,KAAK;GACzB;EACF,CACF;EACA,OAAO,KAAKU;CACd;;;;;;CAOA,sBACE,IACA,KACA,MACA,QACA,OACM;EACN,MAAM,MAAM,KAAK,IAAI;EACrB,KAAKiC,+BAA+B;EACpC,MAAM,gBAAgB,MAAM,KAAKA;EAIjC,GAAG,kBAFD,KAAK,2BACL,oDAC8B,SAAS,WACvC,WAAW,IAAI,GAAG,oBAAoB,OAAO,oBACvB,cAAc,OACpC,GAAI,UAAU,KAAA,IAAY,CAAC,IAAI,CAAC,KAAK,CACvC;CACF;AACF;AAgCA,IAAM,mBAAmB,6BAA6B;AAEtD,IAAM,gBAAgB,KAAK,UAAU,CACnC,YACA,EAAC,KAAK,WAAU,CAClB,CAAoB"}

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts CHANGED Viewed

@@ -104,15 +104,9 @@ export type SubscriberContext = {
      */
     initial: boolean;
     /**
-     * Whether the subscriber's replica writes the SQLite change log that the
-     * `change-streamer` reads for catchup.
-     *
-     * Always `false` from this version on: the change-streamer writes that log
-     * itself, from the stream loop, so no subscriber advances it and no
-     * subscriber's ACK releases the catchup barrier. The parameter stays on the
-     * wire because a subscriber that predates the writer's move still sets it,
-     * and such a subscriber is excluded from SQLite catchup until slice 11 lifts
-     * the exclusion deliberately.
+     * Legacy topology hint retained on the wire. It no longer affects local
+     * routing: the change-streamer writes the SQLite log itself, so no
+     * subscriber's ACK advances its head or releases its catchup barrier.
      */
     logsChangeStream: boolean;
 };
@@ -362,6 +356,7 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
         tableName: v.Type<string>;
         unique: v.Type<boolean>;
         columns: v.Type<Record<string, "ASC" | "DESC">>;
+        predicate: v.Optional<import("../../db/specs.ts").IndexPredicate>;
     }, "schema"> & {
         schema: v.Type<string>;
     }, undefined>;

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AACzD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,UAAU,EAAE,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAClD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;GAOG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;;OAKG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC;CAC1E;AAgCD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB~~;;;;;;;;;;OAUG~~;IACH,gBAAgB,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;aASvB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,SAAS,yDAWrD;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,kEAAkE;AAClE,MAAM,MAAM,oBAAoB,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;AAE1D,MAAM,WAAW,qBACf,SAAQ,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,EAAE,OAAO;IAClD;;;OAGG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAE3D;;;;;;;;OAQG;IACH,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;IAE3E;;;;;;;;;OASG;IACH,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/C"}
1	+ {"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AACzD,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,UAAU,EAAE,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAClD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;GAOG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;;OAKG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC;CAC1E;AAgCD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;;;OAIG;IACH,gBAAgB,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;aASvB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,SAAS,yDAWrD;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,kEAAkE;AAClE,MAAM,MAAM,oBAAoB,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;AAE1D,MAAM,WAAW,qBACf,SAAQ,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,EAAE,OAAO;IAClD;;;OAGG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAE3D;;;;;;;;OAQG;IACH,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;IAE3E;;;;;;;;;OASG;IACH,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/C"}

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {ParsedJSON, Source} from '../../types/streams.ts';\nimport {\n changeStreamDataSchema,\n type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/*\n Internally all downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering while replaying stored changes and\n * filtering changes already seen by a subscriber.\n \n Only commit watermarks are exposed to subscribers. The JSON string is the\n * canonical serialization shared by storage and live forwarding.\n /\nexport type WatermarkedChange = [\n watermark: string,\n tag: ChangeTag,\n json: string,\n];\n\n/\n The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n \n * Changes from the upstream source are immediately forwarded to\n * connected subscribers to minimize latency.\n \n * They are then stored in a separate DB to facilitate catchup\n * of connecting subscribers that are behind.\n \n * Acknowledgements are sent upstream after they are successfully\n * stored.\n \n Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n \n However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n \n The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n /\nexport interface ChangeStreamer {\n /\n Subscribes to changes based on the supplied subscriber `ctx`,\n * which indicates the watermark at which the subscriber is up to\n * date. Each result contains both parsed, validated data and the exact JSON\n * payload sent by the ChangeStreamerService.\n /\n subscribe(ctx: SubscriberContext): Promise<Source<SerializedDownstream>>;\n}\n\n// v1: v0.18\n// - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n// - Adds the \"status\" message which is initially used to signal that the\n// subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n// - Adds the \"taskID\" to the subscription context, and support for\n// the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n// - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n// status request so that a subscriber can verify whether its replica,\n// whether it be restored or existing in a permanent volume, is compatible\n// with the change-streamer.\n// v5: v0.26\n// - Moves relation.keyColumns and relation.replicaIdentity to\n// relation.rowKey: { columns, type }.\n// - Adds `metadata` to `create-table` message\n// - Adds `tableMetadata` to `add-column` message\n// - Adds `table-update-metadata` message\n// v6: v0.26\n// - Adds support for `backfill` messages\n// v6: v1.0.1 (backwards compatible, no version change)\n// - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n// - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n// the upstream commit timestamp for end-to-end serving lag measurement.\n// The stream is parsed in 'passthrough' mode, so an older peer ignores the\n// field, and a newer peer treats its absence (including in changes\n// replayed from the Change DB) as \"no commit time reported\".\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n /\n The supported change-streamer protocol version.\n /\n protocolVersion: number;\n\n /\n Task ID. This is used to link the request with a preceding snapshot\n * reservation.\n /\n taskID: string;\n\n /\n Subscriber id. This is only used for debugging.\n /\n id: string;\n\n /\n The ReplicatorMode of the subscriber. 'backup' indicates that the\n * subscriber is local to the `change-streamer` in the `replication-manager`,\n * while 'serving' indicates that user-facing requests depend on the subscriber.\n /\n mode: ReplicatorMode;\n\n /\n The ChangeStreamer will return an Error if the subscriber is\n * on a different replica version (i.e. the initial snapshot associated\n * with the replication slot).\n /\n replicaVersion: string;\n\n /\n The watermark up to which the subscriber is up to date.\n * Only changes after the watermark will be streamed.\n /\n watermark: string;\n\n /\n Whether this is the first subscription request made by the task,\n * i.e. indicating that the watermark comes from a restored replica\n * backup. The ChangeStreamer uses this to determine which changes\n * are safe to purge from the Storer.\n /\n initial: boolean;\n\n /\n Whether the subscriber's replica writes the SQLite change log that the\n * `change-streamer` reads for catchup.\n \n Always `false` from this version on: the change-streamer writes that log\n * itself, from the stream loop, so no subscriber advances it and no\n * subscriber's ACK releases the catchup barrier. The parameter stays on the\n * wire because a subscriber that predates the writer's move still sets it,\n * and such a subscriber is excluded from SQLite catchup until slice 11 lifts\n * the exclusion deliberately.\n /\n logsChangeStream: boolean;\n};\n\n/\n The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n /\nexport const statusSchema = v.object({\n tag: v.literal('status'),\n\n lagReport: v\n .object({\n lastTimings: changeSourceTimingsSchema.optional(),\n nextSendTimeMs: v.number(),\n })\n .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/\n A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n /\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n type: v.number(), // ErrorType\n message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n statusMessageSchema,\n changeStreamDataSchema,\n errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n switch (val) {\n case ErrorType.WrongReplicaVersion:\n return 'WrongReplicaVersion';\n case ErrorType.WatermarkTooOld:\n return 'WatermarkTooOld';\n case ErrorType.Unknown:\n return 'Unknown';\n default:\n return 'Unknown';\n }\n}\n\n/\n A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n \n A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n /\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\n/* A downstream message and its canonical ChangeStreamer JSON. /\nexport type SerializedDownstream = ParsedJSON<Downstream>;\n\nexport interface ChangeStreamerService\n extends Omit<ChangeStreamer, 'subscribe'>, Service {\n /\n The server-side interface overrides `subscribe()` to return a stream\n * of already-stringified {@link Downstream} payloads.\n /\n subscribe(ctx: SubscriberContext): Promise<Source<string>>;\n\n /\n Starts a snapshot reservation to preserve change-log entries while\n * a soon-to-be-subscriber downloads the current replica backup. Once\n * the change-streamer knows that a backup is available\n * (via {@link trackBackupWatermark}), it sends a confirmation\n * SnapshotMessage and reserves the corresponding entries in the change-log\n * until the client with that `taskID` subscribes, or if the snapshot\n * connection terminates.\n /\n startSnapshotReservation(taskID: string): Promise<Source<SnapshotMessage>>;\n\n /\n Informs the change-streamer of the watermark up to which the replica has\n * been backed up. This serves two purposes:\n * - It serves as the maximum watermark up to which the change-log can be\n * purged. Note that the change-streamer also takes into account the\n * positions of its subscribers and snapshot reservations when purging\n * changes in the change-log.\n * - In RMv2, this ACK's the upstream change-source to allow its buffer of\n * changes (e.g. the replication_slot) to advance.\n */\n trackBackupWatermark(watermark: string): void;\n}\n"],"mappings":";;;;;;;;AA0KA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}
1	+ {"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {ParsedJSON, Source} from '../../types/streams.ts';\nimport {\n changeStreamDataSchema,\n type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/*\n Internally all downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering while replaying stored changes and\n * filtering changes already seen by a subscriber.\n \n Only commit watermarks are exposed to subscribers. The JSON string is the\n * canonical serialization shared by storage and live forwarding.\n /\nexport type WatermarkedChange = [\n watermark: string,\n tag: ChangeTag,\n json: string,\n];\n\n/\n The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n \n * Changes from the upstream source are immediately forwarded to\n * connected subscribers to minimize latency.\n \n * They are then stored in a separate DB to facilitate catchup\n * of connecting subscribers that are behind.\n \n * Acknowledgements are sent upstream after they are successfully\n * stored.\n \n Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n \n However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n \n The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n /\nexport interface ChangeStreamer {\n /\n Subscribes to changes based on the supplied subscriber `ctx`,\n * which indicates the watermark at which the subscriber is up to\n * date. Each result contains both parsed, validated data and the exact JSON\n * payload sent by the ChangeStreamerService.\n /\n subscribe(ctx: SubscriberContext): Promise<Source<SerializedDownstream>>;\n}\n\n// v1: v0.18\n// - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n// - Adds the \"status\" message which is initially used to signal that the\n// subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n// - Adds the \"taskID\" to the subscription context, and support for\n// the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n// - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n// status request so that a subscriber can verify whether its replica,\n// whether it be restored or existing in a permanent volume, is compatible\n// with the change-streamer.\n// v5: v0.26\n// - Moves relation.keyColumns and relation.replicaIdentity to\n// relation.rowKey: { columns, type }.\n// - Adds `metadata` to `create-table` message\n// - Adds `tableMetadata` to `add-column` message\n// - Adds `table-update-metadata` message\n// v6: v0.26\n// - Adds support for `backfill` messages\n// v6: v1.0.1 (backwards compatible, no version change)\n// - Adds lag reporting to status messages\n// v6: (backwards compatible, no version change)\n// - Adds the optional `commitTimeMs` field to `commit` messages, carrying\n// the upstream commit timestamp for end-to-end serving lag measurement.\n// The stream is parsed in 'passthrough' mode, so an older peer ignores the\n// field, and a newer peer treats its absence (including in changes\n// replayed from the Change DB) as \"no commit time reported\".\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n /\n The supported change-streamer protocol version.\n /\n protocolVersion: number;\n\n /\n Task ID. This is used to link the request with a preceding snapshot\n * reservation.\n /\n taskID: string;\n\n /\n Subscriber id. This is only used for debugging.\n /\n id: string;\n\n /\n The ReplicatorMode of the subscriber. 'backup' indicates that the\n * subscriber is local to the `change-streamer` in the `replication-manager`,\n * while 'serving' indicates that user-facing requests depend on the subscriber.\n /\n mode: ReplicatorMode;\n\n /\n The ChangeStreamer will return an Error if the subscriber is\n * on a different replica version (i.e. the initial snapshot associated\n * with the replication slot).\n /\n replicaVersion: string;\n\n /\n The watermark up to which the subscriber is up to date.\n * Only changes after the watermark will be streamed.\n /\n watermark: string;\n\n /\n Whether this is the first subscription request made by the task,\n * i.e. indicating that the watermark comes from a restored replica\n * backup. The ChangeStreamer uses this to determine which changes\n * are safe to purge from the Storer.\n /\n initial: boolean;\n\n /\n Legacy topology hint retained on the wire. It no longer affects local\n * routing: the change-streamer writes the SQLite log itself, so no\n * subscriber's ACK advances its head or releases its catchup barrier.\n /\n logsChangeStream: boolean;\n};\n\n/\n The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n /\nexport const statusSchema = v.object({\n tag: v.literal('status'),\n\n lagReport: v\n .object({\n lastTimings: changeSourceTimingsSchema.optional(),\n nextSendTimeMs: v.number(),\n })\n .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/\n A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n /\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n type: v.number(), // ErrorType\n message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n statusMessageSchema,\n changeStreamDataSchema,\n errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n switch (val) {\n case ErrorType.WrongReplicaVersion:\n return 'WrongReplicaVersion';\n case ErrorType.WatermarkTooOld:\n return 'WatermarkTooOld';\n case ErrorType.Unknown:\n return 'Unknown';\n default:\n return 'Unknown';\n }\n}\n\n/\n A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n \n A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n /\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\n/* A downstream message and its canonical ChangeStreamer JSON. /\nexport type SerializedDownstream = ParsedJSON<Downstream>;\n\nexport interface ChangeStreamerService\n extends Omit<ChangeStreamer, 'subscribe'>, Service {\n /\n The server-side interface overrides `subscribe()` to return a stream\n * of already-stringified {@link Downstream} payloads.\n /\n subscribe(ctx: SubscriberContext): Promise<Source<string>>;\n\n /\n Starts a snapshot reservation to preserve change-log entries while\n * a soon-to-be-subscriber downloads the current replica backup. Once\n * the change-streamer knows that a backup is available\n * (via {@link trackBackupWatermark}), it sends a confirmation\n * SnapshotMessage and reserves the corresponding entries in the change-log\n * until the client with that `taskID` subscribes, or if the snapshot\n * connection terminates.\n /\n startSnapshotReservation(taskID: string): Promise<Source<SnapshotMessage>>;\n\n /\n Informs the change-streamer of the watermark up to which the replica has\n * been backed up. This serves two purposes:\n * - It serves as the maximum watermark up to which the change-log can be\n * purged. Note that the change-streamer also takes into account the\n * positions of its subscribers and snapshot reservations when purging\n * changes in the change-log.\n * - In RMv2, this ACK's the upstream change-source to allow its buffer of\n * changes (e.g. the replication_slot) to advance.\n */\n trackBackupWatermark(watermark: string): void;\n}\n"],"mappings":";;;;;;;;AAoKA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAY,MAAM,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE/E,OAAO,KAAK,EAAC,UAAU,EAAkB,MAAM,iBAAiB,CAAC;AAEjE,MAAM,MAAM,sBAAsB,GAAG;IACnC,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5D,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IAC7C,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF,qBAAa,SAAS;;~~gBAyBlB~~,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAEL;IA+DH,oBAAoB;IAQpB;;;;;;;OAOG;IACH,uBAAuB,CAAC,GAAG,EAAE,MAAM;~~IAyCnC~~,mBAAmB;IAInB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IAKhC,UAAU,CAAC,MAAM,EAAE,MAAM;IAMzB;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB;IAsErD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}
1	+ {"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,KAAK,EAAY,MAAM,EAAE,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAE/E,OAAO,KAAK,EAAC,UAAU,EAAkB,MAAM,iBAAiB,CAAC;AAEjE,MAAM,MAAM,sBAAsB,GAAG;IACnC,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAE5D,YAAY,CAAC,EAAE,OAAO,UAAU,GAAG,SAAS,CAAC;IAC7C,cAAc,CAAC,EAAE,OAAO,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF,qBAAa,SAAS;;gBAgClB,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAEL;IA+DH,oBAAoB;IAQpB;;;;;;;OAOG;IACH,uBAAuB,CAAC,GAAG,EAAE,MAAM;IA+DnC,mBAAmB;IAInB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IAKhC,UAAU,CAAC,MAAM,EAAE,MAAM;IAMzB;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB;IAsErD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}

package/out/zero-cache/src/services/change-streamer/forwarder.js CHANGED Viewed

@@ -15,8 +15,10 @@ var Forwarder = class {
 	#inTransaction = false;
 	#currentBroadcast;
 	#progressMonitor;
+	#backupLagEventCount = 0;
+	#lastBackupLagWarnMs;
 	constructor(lc, opts = { flowControlConsensusTimeoutProportion: 2 }) {
-		this.#lc = lc.withContext("component", "progress-monitor");
+		this.#lc = lc.withContext("component", "subscriber-progress-monitor");
 		this.#setTimeout = opts.setTimeoutFn ?? setTimeout;
 		this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;
 		this.#earlyReleaseOptions = opts.flowControlConsensusTimeoutProportion < 0 ? void 0 : {
@@ -62,8 +64,15 @@ var Forwarder = class {
 		if (this.#flowControlSlowSubscriberGracePeriodMs && timedOut?.size && slowestOnTime) for (const [sub, stats] of timedOut.entries()) {
 			const rate = stats.processRate < slowestOnTime.processRate ? "lagging" : "catching-up";
 			const laggingDuration = sub.reportChangeRate(now, rate);
-			if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {
-				this.#lc.warn?.(`subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`, { stats });
+			if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) if (sub.mode === "backup") {
+				this.#backupLagEventCount++;
+				const grace = this.#flowControlSlowSubscriberGracePeriodMs;
+				if (this.#lastBackupLagWarnMs === void 0 || now - this.#lastBackupLagWarnMs >= grace) {
+					this.#lastBackupLagWarnMs = now;
+					this.#lc.warn?.(`backup subscriber ${sub.id} has lagged for ${laggingDuration}ms (${this.#backupLagEventCount} lag events so far). this is only expected during initial catchup`, { stats });
+				}
+			} else {
+				this.#lc.warn?.(`serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`, { stats });
 				sub.close(3, `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`);
 			}
 		}

package/out/zero-cache/src/services/change-streamer/forwarder.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n getOrCreateCounter,\n getOrCreateGauge,\n getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n flowControlConsensusTimeoutProportion: number;\n flowControlSlowSubscriberGracePeriodMs?: number \| undefined;\n // Injectable timers, for deterministic testing.\n setTimeoutFn?: typeof setTimeout \| undefined;\n clearTimeoutFn?: typeof clearTimeout \| undefined;\n};\n\nexport class Forwarder {\n readonly #lc: LogContext;\n readonly #setTimeout: typeof setTimeout;\n readonly #clearTimeout: typeof clearTimeout;\n readonly #active = new Set<Subscriber>();\n readonly #queued = new Set<Subscriber>();\n readonly #earlyReleaseOptions: EarlyReleaseOptions \| undefined;\n readonly #flowControlSlowSubscriberGracePeriodMs: number \| undefined;\n\n readonly #flowControlWaits = getOrCreateCounter(\n 'replication',\n 'flow_control.waits',\n 'Flow-control checkpoints completed, labeled by release mode.',\n );\n readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n 'replication',\n 'flow_control.wait_duration',\n 'Time replication waits at flow-control checkpoints.',\n );\n #inTransaction = false;\n\n #currentBroadcast: Broadcast \| undefined;\n #progressMonitor: NodeJS.Timeout \| undefined;\n\n constructor(\n lc: LogContext,\n opts: ProgressMonitorOptions = {\n flowControlConsensusTimeoutProportion: 2.0,\n },\n ) {\n this.#lc = lc.withContext('component', 'progress-monitor');\n this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n this.#earlyReleaseOptions =\n opts.flowControlConsensusTimeoutProportion < 0\n ? undefined\n : {\n consensusTimeoutProportion:\n opts.flowControlConsensusTimeoutProportion,\n setTimeoutFn: this.#setTimeout,\n clearTimeoutFn: this.#clearTimeout,\n };\n this.#flowControlSlowSubscriberGracePeriodMs =\n opts.flowControlSlowSubscriberGracePeriodMs;\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.active_subscribers',\n 'Active change-stream subscribers receiving live changes.',\n ).addCallback(result => result.observe(this.#active.size));\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.queued_subscribers',\n 'Change-stream subscribers waiting for the current transaction to finish before activation.',\n ).addCallback(result => result.observe(this.#queued.size));\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.pending_messages',\n 'Downstream change-stream messages not yet acked by subscribers.',\n ).addCallback(result =>\n result.observe(this.#subscriberStats().pendingMessages),\n );\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.backlog_messages',\n 'Live change-stream messages buffered while subscribers catch up.',\n ).addCallback(result =>\n result.observe(this.#subscriberStats().backlogMessages),\n );\n\n getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n description:\n 'Live change-stream bytes buffered while subscribers catch up.',\n unit: 'By',\n }).addCallback(result =>\n result.observe(this.#subscriberStats().backlogBytes),\n );\n\n getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n description:\n 'Maximum live change-stream bytes buffered by a single subscriber.',\n unit: 'By',\n }).addCallback(result =>\n result.observe(this.#subscriberStats().maxBacklogBytes),\n );\n }\n\n startProgressMonitor() {\n clearInterval(this.#progressMonitor);\n this.#progressMonitor = setInterval(\n () => this.checkSubscriberProgress(performance.now()),\n 1000,\n );\n }\n\n /*\n Runs one progress-monitor tick: samples each active subscriber's change\n * rate, logs slow broadcasts, and disconnects subscribers that have been\n * continuously lagging for longer than the configured grace period.\n \n Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n * than private) so tests can drive a single tick with an explicit `now`.\n /\n checkSubscriberProgress(now: number) {\n let timedOut: Map<Subscriber, SubscriberStats> \| undefined;\n let slowestOnTime: SubscriberStats \| undefined;\n\n for (const sub of this.#active) {\n const stats = sub.sampleProcessRate(now).getStats();\n if (stats.missedLastTimeout) {\n (timedOut ??= new Map()).set(sub, stats);\n } else if (\n stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n ) {\n slowestOnTime = stats;\n }\n }\n this.#currentBroadcast?.logSlowRequests(now);\n\n if (\n this.#flowControlSlowSubscriberGracePeriodMs &&\n timedOut?.size &&\n slowestOnTime\n ) {\n for (const [sub, stats] of timedOut.entries()) {\n const rate =\n stats.processRate < slowestOnTime.processRate\n ? 'lagging'\n : 'catching-up';\n const laggingDuration = sub.reportChangeRate(now, rate);\n if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n this.#lc.warn?.(\n `subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n {stats},\n );\n sub.close(\n StreamTooFarBehind,\n `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n );\n }\n }\n }\n }\n\n stopProgressMonitor() {\n clearInterval(this.#progressMonitor);\n }\n\n /\n `add()` is called in lock step with `Storer.catchup()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n /\n add(sub: Subscriber) {\n if (this.#inTransaction) {\n this.#queued.add(sub);\n } else {\n this.#active.add(sub);\n }\n }\n\n remove(sub: Subscriber) {\n this.#active.delete(sub);\n this.#queued.delete(sub);\n sub.close();\n }\n\n /\n `forward()` is called in lockstep with `Storer.store()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n \n This version of forward is fire-and-forget, with no flow control. The\n * change-streamer should call and await {@link forwardWithFlowControl()}\n * occasionally to avoid memory blowup.\n /\n forward(entry: WatermarkedChange) {\n Broadcast.withoutTracking(this.#active.values(), entry);\n this.#updateActiveSubscribers(entry[1]);\n }\n\n sendStatus(status: Status) {\n for (const sub of this.#active.values()) {\n sub.sendStatus(status);\n }\n }\n\n /\n The flow-control-aware equivalent of {@link forward()}, returning a\n * Promise that resolves when replication should continue.\n */\n async forwardWithFlowControl(entry: WatermarkedChange) {\n const start = performance.now();\n const broadcast = new Broadcast(\n this.#lc,\n this.#active.values(),\n entry,\n this.#earlyReleaseOptions,\n );\n this.#updateActiveSubscribers(entry[1]);\n\n // set for progress tracking\n this.#currentBroadcast = broadcast;\n\n try {\n await broadcast.done;\n const attributes = {'release.mode': broadcast.releaseMode};\n this.#flowControlWaits.add(1, attributes);\n this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n } finally {\n // Technically #currentBroadcast may have changed, so only\n // unset if it if is still the same.\n if (this.#currentBroadcast === broadcast) {\n this.#currentBroadcast = undefined;\n }\n }\n }\n\n #subscriberStats() {\n let pendingMessages = 0;\n let backlogMessages = 0;\n let backlogBytes = 0;\n let maxBacklogBytes = 0;\n\n for (const sub of joinIterables(this.#active, this.#queued)) {\n const stats = sub.getStats();\n pendingMessages += stats.pending;\n backlogMessages += stats.backlog;\n backlogBytes += stats.backlogBytes;\n maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n }\n\n return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n }\n\n #updateActiveSubscribers(tag: ChangeTag) {\n switch (tag) {\n case 'begin':\n // While in a Transaction, all added subscribers are \"queued\" so that no\n // messages are forwarded to them. This state corresponds to being queued\n // for catchup in the Storer, which will retrieve historic changes\n // and call catchup() once the current transaction is committed.\n this.#inTransaction = true;\n break;\n case 'commit':\n case 'rollback':\n // Upon commit or rollback, all queued subscribers are transferred to\n // the active set. This means that they can receive messages starting\n // from the next transaction.\n //\n // Note that if catchup is still in progress (in the Storer), these messages\n // will be buffered in the backlog until catchup completes.\n this.#inTransaction = false;\n for (const sub of this.#queued.values()) {\n this.#active.add(sub);\n }\n this.#queued.clear();\n break;\n }\n }\n\n getAcks(): Set<string> {\n return new Set(\n joinIterables(\n wrapIterable(this.#active).map(s => s.acked),\n wrapIterable(this.#queued).map(s => s.acked),\n ),\n );\n }\n}\n"],"mappings":";;;;AAoBA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAAyC;IACnE,KAAKN,IAAI,OACP,cAAc,IAAI,GAAG,kBAAkB,gBAAgB,oBACvD,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EACF;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKE,gBACP,KAAKR,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKU,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKV,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKH,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKQ,yBAAyB,MAAM,EAAE;EAGtC,KAAKF,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKR,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKQ,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKR,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}
1	+ {"version":3,"file":"forwarder.js","names":["#lc","#setTimeout","#clearTimeout","#active","#queued","#earlyReleaseOptions","#flowControlSlowSubscriberGracePeriodMs","#flowControlWaits","#flowControlWaitTime","#subscriberStats","#progressMonitor","#currentBroadcast","#backupLagEventCount","#lastBackupLagWarnMs","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport {\n getOrCreateCounter,\n getOrCreateGauge,\n getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport {Broadcast, type EarlyReleaseOptions} from './broadcast.ts';\nimport type {ChangeTag, Status, WatermarkedChange} from './change-streamer.ts';\nimport {StreamTooFarBehind} from './error-type-enum.ts';\nimport type {Subscriber, SubscriberStats} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n flowControlConsensusTimeoutProportion: number;\n flowControlSlowSubscriberGracePeriodMs?: number \| undefined;\n // Injectable timers, for deterministic testing.\n setTimeoutFn?: typeof setTimeout \| undefined;\n clearTimeoutFn?: typeof clearTimeout \| undefined;\n};\n\nexport class Forwarder {\n readonly #lc: LogContext;\n readonly #setTimeout: typeof setTimeout;\n readonly #clearTimeout: typeof clearTimeout;\n readonly #active = new Set<Subscriber>();\n readonly #queued = new Set<Subscriber>();\n readonly #earlyReleaseOptions: EarlyReleaseOptions \| undefined;\n readonly #flowControlSlowSubscriberGracePeriodMs: number \| undefined;\n\n readonly #flowControlWaits = getOrCreateCounter(\n 'replication',\n 'flow_control.waits',\n 'Flow-control checkpoints completed, labeled by release mode.',\n );\n readonly #flowControlWaitTime = getOrCreateLatencyHistogram(\n 'replication',\n 'flow_control.wait_duration',\n 'Time replication waits at flow-control checkpoints.',\n );\n #inTransaction = false;\n\n #currentBroadcast: Broadcast \| undefined;\n #progressMonitor: NodeJS.Timeout \| undefined;\n\n // The backup-replicator is never disconnected as a laggard, so its lag\n // detections are logged (rather than acted upon). Track the running total and\n // throttle the warning to at most once per grace period to avoid spamming the\n // log for the duration of a slow catchup.\n #backupLagEventCount = 0;\n #lastBackupLagWarnMs: number \| undefined;\n\n constructor(\n lc: LogContext,\n opts: ProgressMonitorOptions = {\n flowControlConsensusTimeoutProportion: 2.0,\n },\n ) {\n this.#lc = lc.withContext('component', 'subscriber-progress-monitor');\n this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n this.#earlyReleaseOptions =\n opts.flowControlConsensusTimeoutProportion < 0\n ? undefined\n : {\n consensusTimeoutProportion:\n opts.flowControlConsensusTimeoutProportion,\n setTimeoutFn: this.#setTimeout,\n clearTimeoutFn: this.#clearTimeout,\n };\n this.#flowControlSlowSubscriberGracePeriodMs =\n opts.flowControlSlowSubscriberGracePeriodMs;\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.active_subscribers',\n 'Active change-stream subscribers receiving live changes.',\n ).addCallback(result => result.observe(this.#active.size));\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.queued_subscribers',\n 'Change-stream subscribers waiting for the current transaction to finish before activation.',\n ).addCallback(result => result.observe(this.#queued.size));\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.pending_messages',\n 'Downstream change-stream messages not yet acked by subscribers.',\n ).addCallback(result =>\n result.observe(this.#subscriberStats().pendingMessages),\n );\n\n getOrCreateGauge(\n 'replication',\n 'flow_control.backlog_messages',\n 'Live change-stream messages buffered while subscribers catch up.',\n ).addCallback(result =>\n result.observe(this.#subscriberStats().backlogMessages),\n );\n\n getOrCreateGauge('replication', 'flow_control.backlog_bytes', {\n description:\n 'Live change-stream bytes buffered while subscribers catch up.',\n unit: 'By',\n }).addCallback(result =>\n result.observe(this.#subscriberStats().backlogBytes),\n );\n\n getOrCreateGauge('replication', 'flow_control.max_backlog_bytes', {\n description:\n 'Maximum live change-stream bytes buffered by a single subscriber.',\n unit: 'By',\n }).addCallback(result =>\n result.observe(this.#subscriberStats().maxBacklogBytes),\n );\n }\n\n startProgressMonitor() {\n clearInterval(this.#progressMonitor);\n this.#progressMonitor = setInterval(\n () => this.checkSubscriberProgress(performance.now()),\n 1000,\n );\n }\n\n /*\n Runs one progress-monitor tick: samples each active subscriber's change\n * rate, logs slow broadcasts, and disconnects subscribers that have been\n * continuously lagging for longer than the configured grace period.\n \n Invoked on an interval by {@link startProgressMonitor()}; exposed (rather\n * than private) so tests can drive a single tick with an explicit `now`.\n /\n checkSubscriberProgress(now: number) {\n let timedOut: Map<Subscriber, SubscriberStats> \| undefined;\n let slowestOnTime: SubscriberStats \| undefined;\n\n for (const sub of this.#active) {\n const stats = sub.sampleProcessRate(now).getStats();\n if (stats.missedLastTimeout) {\n (timedOut ??= new Map()).set(sub, stats);\n } else if (\n stats.processRate <= (slowestOnTime?.processRate ?? Infinity)\n ) {\n slowestOnTime = stats;\n }\n }\n this.#currentBroadcast?.logSlowRequests(now);\n\n if (\n this.#flowControlSlowSubscriberGracePeriodMs &&\n timedOut?.size &&\n slowestOnTime\n ) {\n for (const [sub, stats] of timedOut.entries()) {\n const rate =\n stats.processRate < slowestOnTime.processRate\n ? 'lagging'\n : 'catching-up';\n const laggingDuration = sub.reportChangeRate(now, rate);\n if (laggingDuration >= this.#flowControlSlowSubscriberGracePeriodMs) {\n if (sub.mode === 'backup') {\n // The backup-replicator can only be detected as a laggard during\n // initial catchup; after that it is required for consensus. If it\n // triggers laggard detection in the former case, just log and\n // continue; as disconnecting it would otherwise kill the\n // change-streamer task altogether.\n this.#backupLagEventCount++;\n const grace = this.#flowControlSlowSubscriberGracePeriodMs;\n if (\n this.#lastBackupLagWarnMs === undefined \|\|\n now - this.#lastBackupLagWarnMs >= grace\n ) {\n this.#lastBackupLagWarnMs = now;\n this.#lc.warn?.(\n `backup subscriber ${sub.id} has lagged for ${laggingDuration}ms ` +\n `(${this.#backupLagEventCount} lag events so far). ` +\n `this is only expected during initial catchup`,\n {stats},\n );\n }\n } else {\n this.#lc.warn?.(\n `serving subscriber ${sub.id} has lagged for ${laggingDuration}ms. disconnecting`,\n {stats},\n );\n sub.close(\n StreamTooFarBehind,\n `lagging for over ${this.#flowControlSlowSubscriberGracePeriodMs}ms (${laggingDuration}ms)`,\n );\n }\n }\n }\n }\n }\n\n stopProgressMonitor() {\n clearInterval(this.#progressMonitor);\n }\n\n /\n `add()` is called in lock step with `Storer.catchup()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n /\n add(sub: Subscriber) {\n if (this.#inTransaction) {\n this.#queued.add(sub);\n } else {\n this.#active.add(sub);\n }\n }\n\n remove(sub: Subscriber) {\n this.#active.delete(sub);\n this.#queued.delete(sub);\n sub.close();\n }\n\n /\n `forward()` is called in lockstep with `Storer.store()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n \n This version of forward is fire-and-forget, with no flow control. The\n * change-streamer should call and await {@link forwardWithFlowControl()}\n * occasionally to avoid memory blowup.\n /\n forward(entry: WatermarkedChange) {\n Broadcast.withoutTracking(this.#active.values(), entry);\n this.#updateActiveSubscribers(entry[1]);\n }\n\n sendStatus(status: Status) {\n for (const sub of this.#active.values()) {\n sub.sendStatus(status);\n }\n }\n\n /\n The flow-control-aware equivalent of {@link forward()}, returning a\n * Promise that resolves when replication should continue.\n */\n async forwardWithFlowControl(entry: WatermarkedChange) {\n const start = performance.now();\n const broadcast = new Broadcast(\n this.#lc,\n this.#active.values(),\n entry,\n this.#earlyReleaseOptions,\n );\n this.#updateActiveSubscribers(entry[1]);\n\n // set for progress tracking\n this.#currentBroadcast = broadcast;\n\n try {\n await broadcast.done;\n const attributes = {'release.mode': broadcast.releaseMode};\n this.#flowControlWaits.add(1, attributes);\n this.#flowControlWaitTime.recordMs(performance.now() - start, attributes);\n } finally {\n // Technically #currentBroadcast may have changed, so only\n // unset if it if is still the same.\n if (this.#currentBroadcast === broadcast) {\n this.#currentBroadcast = undefined;\n }\n }\n }\n\n #subscriberStats() {\n let pendingMessages = 0;\n let backlogMessages = 0;\n let backlogBytes = 0;\n let maxBacklogBytes = 0;\n\n for (const sub of joinIterables(this.#active, this.#queued)) {\n const stats = sub.getStats();\n pendingMessages += stats.pending;\n backlogMessages += stats.backlog;\n backlogBytes += stats.backlogBytes;\n maxBacklogBytes = Math.max(maxBacklogBytes, stats.backlogBytes);\n }\n\n return {pendingMessages, backlogMessages, backlogBytes, maxBacklogBytes};\n }\n\n #updateActiveSubscribers(tag: ChangeTag) {\n switch (tag) {\n case 'begin':\n // While in a Transaction, all added subscribers are \"queued\" so that no\n // messages are forwarded to them. This state corresponds to being queued\n // for catchup in the Storer, which will retrieve historic changes\n // and call catchup() once the current transaction is committed.\n this.#inTransaction = true;\n break;\n case 'commit':\n case 'rollback':\n // Upon commit or rollback, all queued subscribers are transferred to\n // the active set. This means that they can receive messages starting\n // from the next transaction.\n //\n // Note that if catchup is still in progress (in the Storer), these messages\n // will be buffered in the backlog until catchup completes.\n this.#inTransaction = false;\n for (const sub of this.#queued.values()) {\n this.#active.add(sub);\n }\n this.#queued.clear();\n break;\n }\n }\n\n getAcks(): Set<string> {\n return new Set(\n joinIterables(\n wrapIterable(this.#active).map(s => s.acked),\n wrapIterable(this.#queued).map(s => s.acked),\n ),\n );\n }\n}\n"],"mappings":";;;;AAoBA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA,0BAAmB,IAAI,IAAgB;CACvC,0BAAmB,IAAI,IAAgB;CACvC;CACA;CAEA,oBAA6B,mBAC3B,eACA,sBACA,8DACF;CACA,uBAAgC,4BAC9B,eACA,8BACA,qDACF;CACA,iBAAiB;CAEjB;CACA;CAMA,uBAAuB;CACvB;CAEA,YACE,IACA,OAA+B,EAC7B,uCAAuC,EACzC,GACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,6BAA6B;EACpE,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,KAAKG,uBACH,KAAK,wCAAwC,IACzC,KAAA,IACA;GACE,4BACE,KAAK;GACP,cAAc,KAAKJ;GACnB,gBAAgB,KAAKC;EACvB;EACN,KAAKI,0CACH,KAAK;EAEP,iBACE,eACA,mCACA,0DACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKH,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,mCACA,4FACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,QAAQ,IAAI,CAAC;EAEzD,iBACE,eACA,iCACA,iEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKK,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBACE,eACA,iCACA,kEACF,EAAE,aAAY,WACZ,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;EAEA,iBAAiB,eAAe,8BAA8B;GAC5D,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,YAAY,CACrD;EAEA,iBAAiB,eAAe,kCAAkC;GAChE,aACE;GACF,MAAM;EACR,CAAC,EAAE,aAAY,WACb,OAAO,QAAQ,KAAKA,iBAAiB,EAAE,eAAe,CACxD;CACF;CAEA,uBAAuB;EACrB,cAAc,KAAKC,gBAAgB;EACnC,KAAKA,mBAAmB,kBAChB,KAAK,wBAAwB,YAAY,IAAI,CAAC,GACpD,GACF;CACF;;;;;;;;;CAUA,wBAAwB,KAAa;EACnC,IAAI;EACJ,IAAI;EAEJ,KAAK,MAAM,OAAO,KAAKP,SAAS;GAC9B,MAAM,QAAQ,IAAI,kBAAkB,GAAG,EAAE,SAAS;GAClD,IAAI,MAAM,mBACR,CAAC,6BAAa,IAAI,IAAI,GAAG,IAAI,KAAK,KAAK;QAClC,IACL,MAAM,gBAAgB,eAAe,eAAe,WAEpD,gBAAgB;EAEpB;EACA,KAAKQ,mBAAmB,gBAAgB,GAAG;EAE3C,IACE,KAAKL,2CACL,UAAU,QACV,eAEA,KAAK,MAAM,CAAC,KAAK,UAAU,SAAS,QAAQ,GAAG;GAC7C,MAAM,OACJ,MAAM,cAAc,cAAc,cAC9B,YACA;GACN,MAAM,kBAAkB,IAAI,iBAAiB,KAAK,IAAI;GACtD,IAAI,mBAAmB,KAAKA,yCAC1B,IAAI,IAAI,SAAS,UAAU;IAMzB,KAAKM;IACL,MAAM,QAAQ,KAAKN;IACnB,IACE,KAAKO,yBAAyB,KAAA,KAC9B,MAAM,KAAKA,wBAAwB,OACnC;KACA,KAAKA,uBAAuB;KAC5B,KAAKb,IAAI,OACP,qBAAqB,IAAI,GAAG,kBAAkB,gBAAgB,MACxD,KAAKY,qBAAqB,oEAEhC,EAAC,MAAK,CACR;IACF;GACF,OAAO;IACL,KAAKZ,IAAI,OACP,sBAAsB,IAAI,GAAG,kBAAkB,gBAAgB,oBAC/D,EAAC,MAAK,CACR;IACA,IAAI,MAAA,GAEF,oBAAoB,KAAKM,wCAAwC,MAAM,gBAAgB,IACzF;GACF;EAEJ;CAEJ;CAEA,sBAAsB;EACpB,cAAc,KAAKI,gBAAgB;CACrC;;;;;;CAOA,IAAI,KAAiB;EACnB,IAAI,KAAKI,gBACP,KAAKV,QAAQ,IAAI,GAAG;OAEpB,KAAKD,QAAQ,IAAI,GAAG;CAExB;CAEA,OAAO,KAAiB;EACtB,KAAKA,QAAQ,OAAO,GAAG;EACvB,KAAKC,QAAQ,OAAO,GAAG;EACvB,IAAI,MAAM;CACZ;;;;;;;;;;CAWA,QAAQ,OAA0B;EAChC,UAAU,gBAAgB,KAAKD,QAAQ,OAAO,GAAG,KAAK;EACtD,KAAKY,yBAAyB,MAAM,EAAE;CACxC;CAEA,WAAW,QAAgB;EACzB,KAAK,MAAM,OAAO,KAAKZ,QAAQ,OAAO,GACpC,IAAI,WAAW,MAAM;CAEzB;;;;;CAMA,MAAM,uBAAuB,OAA0B;EACrD,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,YAAY,IAAI,UACpB,KAAKH,KACL,KAAKG,QAAQ,OAAO,GACpB,OACA,KAAKE,oBACP;EACA,KAAKU,yBAAyB,MAAM,EAAE;EAGtC,KAAKJ,oBAAoB;EAEzB,IAAI;GACF,MAAM,UAAU;GAChB,MAAM,aAAa,EAAC,gBAAgB,UAAU,YAAW;GACzD,KAAKJ,kBAAkB,IAAI,GAAG,UAAU;GACxC,KAAKC,qBAAqB,SAAS,YAAY,IAAI,IAAI,OAAO,UAAU;EAC1E,UAAU;GAGR,IAAI,KAAKG,sBAAsB,WAC7B,KAAKA,oBAAoB,KAAA;EAE7B;CACF;CAEA,mBAAmB;EACjB,IAAI,kBAAkB;EACtB,IAAI,kBAAkB;EACtB,IAAI,eAAe;EACnB,IAAI,kBAAkB;EAEtB,KAAK,MAAM,OAAO,cAAc,KAAKR,SAAS,KAAKC,OAAO,GAAG;GAC3D,MAAM,QAAQ,IAAI,SAAS;GAC3B,mBAAmB,MAAM;GACzB,mBAAmB,MAAM;GACzB,gBAAgB,MAAM;GACtB,kBAAkB,KAAK,IAAI,iBAAiB,MAAM,YAAY;EAChE;EAEA,OAAO;GAAC;GAAiB;GAAiB;GAAc;EAAe;CACzE;CAEA,yBAAyB,KAAgB;EACvC,QAAQ,KAAR;GACE,KAAK;IAKH,KAAKU,iBAAiB;IACtB;GACF,KAAK;GACL,KAAK;IAOH,KAAKA,iBAAiB;IACtB,KAAK,MAAM,OAAO,KAAKV,QAAQ,OAAO,GACpC,KAAKD,QAAQ,IAAI,GAAG;IAEtB,KAAKC,QAAQ,MAAM;IACnB;EACJ;CACF;CAEA,UAAuB;EACrB,OAAO,IAAI,IACT,cACE,aAAa,KAAKD,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,GAC3C,aAAa,KAAKC,OAAO,EAAE,KAAI,MAAK,EAAE,KAAK,CAC7C,CACF;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"litestream3-prometheus-poller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;~~AAKjD~~,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAOnD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAE3D,eAAO,MAAM,iBAAiB,QAAS,CAAC;AAExC;;;GAGG;AACH,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,wBAAwB,QAAc,CAAC;AAIpD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAG5D;;;;GAIG;AACH,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAEzD;;;;;;;GAOG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAEtD;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,2BAA2B;;gBAwCpC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,iBAAiB,EAAE,mBAAmB;IAexC;;;OAGG;IACH,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC;IA8BlC,QAAQ,CAAC,+BAA+B,sBAgBtC;~~CA2RH~~"}
1	+ {"version":3,"file":"litestream3-prometheus-poller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAOnD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAE3D,eAAO,MAAM,iBAAiB,QAAS,CAAC;AAExC;;;GAGG;AACH,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,wBAAwB,QAAc,CAAC;AAIpD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAG5D;;;;GAIG;AACH,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAEzD;;;;;;;GAOG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAEtD;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,2BAA2B;;gBAwCpC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,iBAAiB,EAAE,mBAAmB;IAexC;;;OAGG;IACH,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC;IA8BlC,QAAQ,CAAC,+BAA+B,sBAgBtC;CAmSH"}