npm - @rocicorp/zero - Versions diffs - 1.0.0 → 1.0.1-canary.0 - Mend

@rocicorp/zero 1.0.0 → 1.0.1-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (75) hide show

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#stream","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n min,\n type AtLeastOne,\n type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n ChangeSource,\n ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n type ChangeStreamControl,\n type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n publishReplicationError,\n replicationStatusError,\n type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n DEFAULT_MAX_RETRY_DELAY_MS,\n RunningState,\n UnrecoverableError,\n} from '../running-state.ts';\nimport {\n type ChangeStreamerService,\n type Downstream,\n type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n AutoResetSignal,\n ensureReplicationConfig,\n markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/*\n Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n /\nexport async function initializeStreamer(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n changeSource: ChangeSource,\n replicationStatusPublisher: ReplicationStatusPublisher,\n subscriptionState: SubscriptionState,\n autoReset: boolean,\n backPressureLimitHeapProportion: number,\n flowControlConsensusPaddingSeconds: number,\n setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n // Make sure the ChangeLog DB is set up.\n await initChangeStreamerSchema(lc, changeDB, shard);\n await ensureReplicationConfig(\n lc,\n changeDB,\n subscriptionState,\n shard,\n autoReset,\n setTimeoutFn,\n );\n\n const {replicaVersion} = subscriptionState;\n return new ChangeStreamerImpl(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n changeSource,\n replicationStatusPublisher,\n autoReset,\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n setTimeoutFn,\n );\n}\n\n/\n Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n \n However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n /\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/\n Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n \n ### Subscriber Catchup\n \n Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n \n \n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * \| Historic changes in storage \| Pending (streamed) tx \| Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * Replication stream\n * > > > > > > > > >\n * ^ ---> required catchup ---> ^\n * Subscriber watermark Subscription begins\n * ```\n \n Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n \n Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n \n * If the streamer is in the middle of a pending Transaction, the\n * Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n * state, new changes are not forwarded to the Subscriber, and catchup\n * is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n * by the Storer, it begins catchup on the Subscriber (with a READONLY\n * snapshot so that it does not block subsequent storage operations).\n * This catchup is thus guaranteed to load the change log entries of\n * that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n * Subscriber from the \"queued\" to the \"active\" set of clients such that\n * the Subscriber begins receiving new changes, starting from the next\n * Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n * is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n * and the Subscriber henceforth forwards changes as they are received.\n \n In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n \n ### Watermarks and ordering\n \n The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n \n https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n \n > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n \n In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n \n A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n \n ```\n * LSN: 1 2 3 4 5 6 7 8 9 10\n * tx1: begin data data data commit\n * tx2: begin data data data commit\n * ```\n \n will arrive as:\n \n ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n \n Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n \n Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n \n * A `commit` message is assigned to a watermark corresponding to its LSN.\n * These are guaranteed to be in commit order by definition.\n \n * `begin` and `data` messages are assigned to the watermark of the\n * preceding `commit` (the previous transaction, or the replication\n * slot's starting LSN) plus 1. This guarantees that they will be sorted\n * after the previously commit transaction even if their LSNs came before it.\n * This is referred to as the `preCommitWatermark`.\n \n * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n * the position of the message within its transaction, with the `begin` message\n * starting at `0`. This guarantees that `begin` and `data` messages will be\n * fetched in the original ChangeSource order during catchup.\n \n `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n \n ### Cleanup\n \n As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n \n More concretely:\n \n * The `initial`, backup-derived watermark is the earliest to which cleanup\n * should ever happen.\n \n * However, it is possible for the replica backup to be ahead of a connected\n * subscriber; and if a network error causes that subscriber to retry from its\n * last watermark, the change streamer must support it.\n \n Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n /\nclass ChangeStreamerImpl implements ChangeStreamerService {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #replicaVersion: string;\n readonly #source: ChangeSource;\n readonly #storer: Storer;\n readonly #forwarder: Forwarder;\n readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n readonly #autoReset: boolean;\n readonly #state: RunningState;\n readonly #initialWatermarks = new Set<string>();\n\n // Starting the (Postgres) ChangeStream results in killing the previous\n // Postgres subscriber, potentially creating a gap in which the old\n // change-streamer has shut down and the new change-streamer has not yet\n // been recognized as \"healthy\" (and thus does not get any requests).\n //\n // To minimize this gap, delay starting the ChangeStream until the first\n // request from a `serving` replicator, indicating that higher level\n // load-balancing / routing logic has begun routing requests to this task.\n readonly #serving = resolver();\n\n readonly #txCounter = getOrCreateCounter(\n 'replication',\n 'transactions',\n 'Count of replicated transactions',\n );\n\n #stream: ChangeStream \| undefined;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n replicaVersion: string,\n source: ChangeSource,\n replicationStatusPublisher: ReplicationStatusPublisher,\n autoReset: boolean,\n backPressureLimitHeapProportion: number,\n flowControlConsensusPaddingSeconds: number,\n setTimeoutFn = setTimeout,\n ) {\n this.id = `change-streamer`;\n this.#lc = lc.withContext('component', 'change-streamer');\n this.#shard = shard;\n this.#changeDB = changeDB;\n this.#replicaVersion = replicaVersion;\n this.#source = source;\n this.#storer = new Storer(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n err => this.stop(err),\n backPressureLimitHeapProportion,\n );\n this.#forwarder = new Forwarder(lc, {\n flowControlConsensusPaddingSeconds,\n });\n this.#replicationStatusPublisher = replicationStatusPublisher;\n this.#autoReset = autoReset;\n this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n }\n\n async run() {\n this.#lc.info?.('starting change stream');\n\n this.#forwarder.startProgressMonitor();\n\n // Once this change-streamer acquires \"ownership\" of the change DB,\n // it is safe to start the storer.\n await this.#storer.assumeOwnership();\n\n // The threshold in (estimated number of) bytes to send() on subscriber\n // websockets before `await`-ing the I/O buffers to be ready for more.\n const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n while (this.#state.shouldRun()) {\n let err: unknown;\n let watermark: string \| null = null;\n let unflushedBytes = 0;\n try {\n const {lastWatermark, backfillRequests} =\n await this.#storer.getStartStreamInitializationParameters();\n const stream = await this.#source.startStream(\n lastWatermark,\n backfillRequests,\n );\n this.#storer.run().catch(e => stream.changes.cancel(e));\n\n this.#stream = stream;\n this.#state.resetBackoff();\n this.#replicationStatusPublisher.publish(\n this.#lc,\n 'Replicating',\n `Replicating from ${lastWatermark}`,\n );\n watermark = null;\n\n for await (const change of stream.changes) {\n const [type, msg] = change;\n switch (type) {\n case 'status':\n if (msg.ack) {\n this.#storer.status(change); // storer acks once it gets through its queue\n }\n continue;\n case 'control':\n await this.#handleControlMessage(msg);\n continue; // control messages are not stored/forwarded\n case 'begin':\n watermark = change[2].commitWatermark;\n break;\n case 'commit':\n if (watermark !== change[2].watermark) {\n throw new UnrecoverableError(\n `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n );\n }\n this.#txCounter.add(1);\n break;\n default:\n if (watermark === null) {\n throw new UnrecoverableError(\n `${type} change (${msg.tag}) received before 'begin' message`,\n );\n }\n break;\n }\n\n const entry: WatermarkedChange = [watermark, change];\n unflushedBytes += this.#storer.store(entry);\n if (unflushedBytes < flushBytesThreshold) {\n // pipeline changes until flushBytesThreshold\n this.#forwarder.forward(entry);\n } else {\n // Wait for messages to clear socket buffers to ensure that they\n // make their way to subscribers. Without this `await`, the\n // messages end up being buffered in this process, which:\n // (1) results in memory pressure and increased GC activity\n // (2) prevents subscribers from processing the messages as they\n // arrive, instead getting them in a large batch after being\n // idle while they were queued (causing further delays).\n await this.#forwarder.forwardWithFlowControl(entry);\n unflushedBytes = 0;\n }\n\n if (type === 'commit' \|\| type === 'rollback') {\n watermark = null;\n }\n\n // Allow the storer to exert back pressure.\n const readyForMore = this.#storer.readyForMore();\n if (readyForMore) {\n await readyForMore;\n }\n }\n } catch (e) {\n err = e;\n } finally {\n this.#stream?.changes.cancel();\n this.#stream = undefined;\n }\n\n // When the change stream is interrupted, abort any pending transaction.\n if (watermark) {\n this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n this.#storer.abort();\n await this.#forwarder.forward([\n watermark,\n ['rollback', {tag: 'rollback'}],\n ]);\n }\n\n // Backoff and drain any pending entries in the storer before reconnecting.\n await Promise.all([\n this.#storer.stop(),\n this.#state.backoff(this.#lc, err),\n this.#state.retryDelay > 5000\n ? publishCriticalEvent(\n this.#lc,\n replicationStatusError(this.#lc, 'Replicating', err),\n )\n : promiseVoid,\n ]);\n }\n\n this.#forwarder.stopProgressMonitor();\n this.#lc.info?.('ChangeStreamer stopped');\n }\n\n async #handleControlMessage(msg: ChangeStreamControl[1]) {\n this.#lc.info?.('received control message', msg);\n const {tag} = msg;\n\n switch (tag) {\n case 'reset-required':\n await markResetRequired(this.#changeDB, this.#shard);\n await publishReplicationError(\n this.#lc,\n 'Replicating',\n msg.message ?? 'Resync required',\n msg.errorDetails,\n );\n if (this.#autoReset) {\n this.#lc.warn?.('shutting down for auto-reset');\n await this.stop(new AutoResetSignal());\n }\n break;\n default:\n unreachable(tag);\n }\n }\n\n subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n if (mode === 'serving') {\n this.#serving.resolve();\n }\n const downstream = Subscription.create<Downstream>({\n cleanup: () => this.#forwarder.remove(subscriber),\n });\n const subscriber = new Subscriber(\n protocolVersion,\n id,\n watermark,\n downstream,\n );\n if (replicaVersion !== this.#replicaVersion) {\n this.#lc.warn?.(\n `rejecting subscriber at replica version ${replicaVersion}`,\n );\n subscriber.close(\n ErrorType.WrongReplicaVersion,\n `current replica version is ${\n this.#replicaVersion\n } (requested ${replicaVersion})`,\n );\n } else {\n this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n this.#forwarder.add(subscriber);\n this.#storer.catchup(subscriber, mode);\n\n if (initial) {\n this.scheduleCleanup(watermark);\n }\n }\n return Promise.resolve(downstream);\n }\n\n scheduleCleanup(watermark: string) {\n const origSize = this.#initialWatermarks.size;\n this.#initialWatermarks.add(watermark);\n\n if (origSize === 0) {\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n\n async getChangeLogState(): Promise<{\n replicaVersion: string;\n minWatermark: string;\n }> {\n const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n if (!minWatermark) {\n this.#lc.warn?.(\n `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n );\n }\n return {\n replicaVersion: this.#replicaVersion,\n minWatermark: minWatermark ?? this.#replicaVersion,\n };\n }\n\n /\n Makes a best effort to purge the change log. In the event of a database\n * error, exceptions will be logged and swallowed, so this method is safe\n * to run in a timeout.\n /\n async #purgeOldChanges(): Promise<void> {\n const initial = [...this.#initialWatermarks];\n if (initial.length === 0) {\n this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n return;\n }\n const current = [...this.#forwarder.getAcks()];\n if (current.length === 0) {\n // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n // Bail to be safe.\n this.#lc.warn?.('No subscribers to confirm cleanup');\n return;\n }\n try {\n const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n if (earliestCurrent < earliestInitial) {\n this.#lc.info?.(\n `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n );\n } else {\n this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n const start = performance.now();\n const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n );\n this.#initialWatermarks.delete(earliestInitial);\n }\n } catch (e) {\n this.#lc.warn?.(`error purging change log`, e);\n } finally {\n if (this.#initialWatermarks.size) {\n // If there are unpurged watermarks to check, schedule the next purge.\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n }\n\n async stop(err?: unknown) {\n this.#state.stop(this.#lc, err);\n this.#stream?.changes.cancel();\n await this.#storer.stop();\n }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n// `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS 3;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAsDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,iCACA,oCACA,eAAe,YACiB;AAEhC,OAAM,yBAAyB,IAAI,UAAU,MAAM;AACnD,OAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,aACD;CAED,MAAM,EAAC,mBAAkB;AACzB,QAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,iCACA,oCACA,aACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyJH,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,KAAa;CAU/C,WAAoB,UAAU;CAE9B,aAAsB,mBACpB,eACA,gBACA,mCACD;CAED;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,WACA,iCACA,oCACA,eAAe,YACf;AACA,OAAK,KAAK;AACV,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AACzD,QAAA,QAAc;AACd,QAAA,WAAiB;AACjB,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,SAAe,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,MAAA,QAAc,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;GAAG,CAAC,GACzE,QAAO,KAAK,KAAK,IAAI,EACrB,gCACD;AACD,QAAA,YAAkB,IAAI,UAAU,IAAI,EAClC,oCACD,CAAC;AACF,QAAA,6BAAmC;AACnC,QAAA,YAAkB;AAClB,QAAA,QAAc,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,aAAa;;CAGlE,MAAM,MAAM;AACV,QAAA,GAAS,OAAO,yBAAyB;AAEzC,QAAA,UAAgB,sBAAsB;AAItC,QAAM,MAAA,OAAa,iBAAiB;EAIpC,MAAM,sBAAsB,wBAAwB,MAAM;AAE1D,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;AACrB,OAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,MAAA,OAAa,wCAAwC;IAC7D,MAAM,SAAS,MAAM,MAAA,OAAa,YAChC,eACA,iBACD;AACD,UAAA,OAAa,KAAK,CAAC,OAAM,MAAK,OAAO,QAAQ,OAAO,EAAE,CAAC;AAEvD,UAAA,SAAe;AACf,UAAA,MAAY,cAAc;AAC1B,UAAA,2BAAiC,QAC/B,MAAA,IACA,eACA,oBAAoB,gBACrB;AACD,gBAAY;AAEZ,eAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;AACpB,aAAQ,MAAR;MACE,KAAK;AACH,WAAI,IAAI,IACN,OAAA,OAAa,OAAO,OAAO;AAE7B;MACF,KAAK;AACH,aAAM,MAAA,qBAA2B,IAAI;AACrC;MACF,KAAK;AACH,mBAAY,OAAO,GAAG;AACtB;MACF,KAAK;AACH,WAAI,cAAc,OAAO,GAAG,UAC1B,OAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,YAC7E;AAEH,aAAA,UAAgB,IAAI,EAAE;AACtB;MACF;AACE,WAAI,cAAc,KAChB,OAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,mCAC5B;AAEH;;KAGJ,MAAM,QAA2B,CAAC,WAAW,OAAO;AACpD,uBAAkB,MAAA,OAAa,MAAM,MAAM;AAC3C,SAAI,iBAAiB,oBAEnB,OAAA,UAAgB,QAAQ,MAAM;UACzB;AAQL,YAAM,MAAA,UAAgB,uBAAuB,MAAM;AACnD,uBAAiB;;AAGnB,SAAI,SAAS,YAAY,SAAS,WAChC,aAAY;KAId,MAAM,eAAe,MAAA,OAAa,cAAc;AAChD,SAAI,aACF,OAAM;;YAGH,GAAG;AACV,UAAM;aACE;AACR,UAAA,QAAc,QAAQ,QAAQ;AAC9B,UAAA,SAAe,KAAA;;AAIjB,OAAI,WAAW;AACb,UAAA,GAAS,OAAO,oCAAoC,YAAY;AAChE,UAAA,OAAa,OAAO;AACpB,UAAM,MAAA,UAAgB,QAAQ,CAC5B,WACA,CAAC,YAAY,EAAC,KAAK,YAAW,CAAC,CAChC,CAAC;;AAIJ,SAAM,QAAQ,IAAI;IAChB,MAAA,OAAa,MAAM;IACnB,MAAA,MAAY,QAAQ,MAAA,IAAU,IAAI;IAClC,MAAA,MAAY,aAAa,MACrB,qBACE,MAAA,IACA,uBAAuB,MAAA,IAAU,eAAe,IAAI,CACrD,GACD;IACL,CAAC;;AAGJ,QAAA,UAAgB,qBAAqB;AACrC,QAAA,GAAS,OAAO,yBAAyB;;CAG3C,OAAA,qBAA4B,KAA6B;AACvD,QAAA,GAAS,OAAO,4BAA4B,IAAI;EAChD,MAAM,EAAC,QAAO;AAEd,UAAQ,KAAR;GACE,KAAK;AACH,UAAM,kBAAkB,MAAA,UAAgB,MAAA,MAAY;AACpD,UAAM,wBACJ,MAAA,IACA,eACA,IAAI,WAAW,mBACf,IAAI,aACL;AACD,QAAI,MAAA,WAAiB;AACnB,WAAA,GAAS,OAAO,+BAA+B;AAC/C,WAAM,KAAK,KAAK,IAAI,iBAAiB,CAAC;;AAExC;GACF,QACE,aAAY,IAAI;;;CAItB,UAAU,KAAqD;EAC7D,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,MAAI,SAAS,UACX,OAAA,QAAc,SAAS;EAEzB,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe,MAAA,UAAgB,OAAO,WAAW,EAClD,CAAC;EACF,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,WACD;AACD,MAAI,mBAAmB,MAAA,gBAAsB;AAC3C,SAAA,GAAS,OACP,2CAA2C,iBAC5C;AACD,cAAW,MACT,GACA,8BACE,MAAA,eACD,cAAc,eAAe,GAC/B;SACI;AACL,SAAA,GAAS,QAAQ,qBAAqB,WAAW,KAAK;AAEtD,SAAA,UAAgB,IAAI,WAAW;AAC/B,SAAA,OAAa,QAAQ,YAAY,KAAK;AAEtC,OAAI,QACF,MAAK,gBAAgB,UAAU;;AAGnC,SAAO,QAAQ,QAAQ,WAAW;;CAGpC,gBAAgB,WAAmB;EACjC,MAAM,WAAW,MAAA,kBAAwB;AACzC,QAAA,kBAAwB,IAAI,UAAU;AAEtC,MAAI,aAAa,EACf,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;CAI3E,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,MAAA,OAAa,2BAA2B;AACnE,MAAI,CAAC,aACH,OAAA,GAAS,OACP,+EACD;AAEH,SAAO;GACL,gBAAgB,MAAA;GAChB,cAAc,gBAAgB,MAAA;GAC/B;;;;;;;CAQH,OAAA,kBAAwC;EACtC,MAAM,UAAU,CAAC,GAAG,MAAA,kBAAwB;AAC5C,MAAI,QAAQ,WAAW,GAAG;AACxB,SAAA,GAAS,OAAO,6CAA6C;AAC7D;;EAEF,MAAM,UAAU,CAAC,GAAG,MAAA,UAAgB,SAAS,CAAC;AAC9C,MAAI,QAAQ,WAAW,GAAG;AAGxB,SAAA,GAAS,OAAO,oCAAoC;AACpD;;AAEF,MAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,QAAoC;GACpE,MAAM,kBAAkB,IAAI,GAAI,QAAoC;AACpE,OAAI,kBAAkB,gBACpB,OAAA,GAAS,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,GAC/E;QACI;AACL,UAAA,GAAS,OAAO,0BAA0B,gBAAgB,MAAM;IAChE,MAAM,QAAQ,YAAY,KAAK;IAC/B,MAAM,UAAU,MAAM,MAAA,OAAa,mBAAmB,gBAAgB;IACtE,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,UAAA,GAAS,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,MACjE;AACD,UAAA,kBAAwB,OAAO,gBAAgB;;WAE1C,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;YACtC;AACR,OAAI,MAAA,kBAAwB,KAE1B,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;;CAK7E,MAAM,KAAK,KAAe;AACxB,QAAA,MAAY,KAAK,MAAA,IAAU,IAAI;AAC/B,QAAA,QAAc,QAAQ,QAAQ;AAC9B,QAAM,MAAA,OAAa,MAAM;;;AAmB7B,IAAM,mBAAmB,6BAA6B"}
1	+ {"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#stream","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n min,\n type AtLeastOne,\n type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n ChangeSource,\n ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n type ChangeStreamControl,\n type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n publishReplicationError,\n replicationStatusError,\n type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n DEFAULT_MAX_RETRY_DELAY_MS,\n RunningState,\n UnrecoverableError,\n} from '../running-state.ts';\nimport {\n type ChangeStreamerService,\n type Downstream,\n type Status,\n type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n AutoResetSignal,\n ensureReplicationConfig,\n markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/*\n Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n /\nexport async function initializeStreamer(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n changeSource: ChangeSource,\n replicationStatusPublisher: ReplicationStatusPublisher,\n subscriptionState: SubscriptionState,\n autoReset: boolean,\n backPressureLimitHeapProportion: number,\n flowControlConsensusPaddingSeconds: number,\n setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n // Make sure the ChangeLog DB is set up.\n await initChangeStreamerSchema(lc, changeDB, shard);\n await ensureReplicationConfig(\n lc,\n changeDB,\n subscriptionState,\n shard,\n autoReset,\n setTimeoutFn,\n );\n\n const {replicaVersion} = subscriptionState;\n return new ChangeStreamerImpl(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n changeSource,\n replicationStatusPublisher,\n autoReset,\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n setTimeoutFn,\n );\n}\n\n/\n Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n \n However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n /\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/\n Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n \n ### Subscriber Catchup\n \n Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n \n \n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * \| Historic changes in storage \| Pending (streamed) tx \| Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * Replication stream\n * > > > > > > > > >\n * ^ ---> required catchup ---> ^\n * Subscriber watermark Subscription begins\n * ```\n \n Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n \n Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n \n * If the streamer is in the middle of a pending Transaction, the\n * Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n * state, new changes are not forwarded to the Subscriber, and catchup\n * is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n * by the Storer, it begins catchup on the Subscriber (with a READONLY\n * snapshot so that it does not block subsequent storage operations).\n * This catchup is thus guaranteed to load the change log entries of\n * that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n * Subscriber from the \"queued\" to the \"active\" set of clients such that\n * the Subscriber begins receiving new changes, starting from the next\n * Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n * is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n * and the Subscriber henceforth forwards changes as they are received.\n \n In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n \n ### Watermarks and ordering\n \n The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n \n https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n \n > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n \n In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n \n A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n \n ```\n * LSN: 1 2 3 4 5 6 7 8 9 10\n * tx1: begin data data data commit\n * tx2: begin data data data commit\n * ```\n \n will arrive as:\n \n ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n \n Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n \n Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n \n * A `commit` message is assigned to a watermark corresponding to its LSN.\n * These are guaranteed to be in commit order by definition.\n \n * `begin` and `data` messages are assigned to the watermark of the\n * preceding `commit` (the previous transaction, or the replication\n * slot's starting LSN) plus 1. This guarantees that they will be sorted\n * after the previously commit transaction even if their LSNs came before it.\n * This is referred to as the `preCommitWatermark`.\n \n * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n * the position of the message within its transaction, with the `begin` message\n * starting at `0`. This guarantees that `begin` and `data` messages will be\n * fetched in the original ChangeSource order during catchup.\n \n `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n \n ### Cleanup\n \n As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n \n More concretely:\n \n * The `initial`, backup-derived watermark is the earliest to which cleanup\n * should ever happen.\n \n * However, it is possible for the replica backup to be ahead of a connected\n * subscriber; and if a network error causes that subscriber to retry from its\n * last watermark, the change streamer must support it.\n \n Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n /\nclass ChangeStreamerImpl implements ChangeStreamerService {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #replicaVersion: string;\n readonly #source: ChangeSource;\n readonly #storer: Storer;\n readonly #forwarder: Forwarder;\n readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n readonly #autoReset: boolean;\n readonly #state: RunningState;\n readonly #initialWatermarks = new Set<string>();\n\n // Starting the (Postgres) ChangeStream results in killing the previous\n // Postgres subscriber, potentially creating a gap in which the old\n // change-streamer has shut down and the new change-streamer has not yet\n // been recognized as \"healthy\" (and thus does not get any requests).\n //\n // To minimize this gap, delay starting the ChangeStream until the first\n // request from a `serving` replicator, indicating that higher level\n // load-balancing / routing logic has begun routing requests to this task.\n readonly #serving = resolver();\n\n readonly #txCounter = getOrCreateCounter(\n 'replication',\n 'transactions',\n 'Count of replicated transactions',\n );\n\n #latestStatus: Status;\n #stream: ChangeStream \| undefined;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n replicaVersion: string,\n source: ChangeSource,\n replicationStatusPublisher: ReplicationStatusPublisher,\n autoReset: boolean,\n backPressureLimitHeapProportion: number,\n flowControlConsensusPaddingSeconds: number,\n setTimeoutFn = setTimeout,\n ) {\n this.id = `change-streamer`;\n this.#lc = lc.withContext('component', 'change-streamer');\n this.#shard = shard;\n this.#changeDB = changeDB;\n this.#replicaVersion = replicaVersion;\n this.#source = source;\n this.#storer = new Storer(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n err => this.stop(err),\n backPressureLimitHeapProportion,\n );\n this.#forwarder = new Forwarder(lc, {\n flowControlConsensusPaddingSeconds,\n });\n this.#replicationStatusPublisher = replicationStatusPublisher;\n this.#autoReset = autoReset;\n this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n this.#latestStatus = {tag: 'status'};\n }\n\n async run() {\n this.#lc.info?.('starting change stream');\n\n this.#forwarder.startProgressMonitor();\n\n const lagReport = await this.#source.startLagReporter();\n if (lagReport) {\n this.#latestStatus.lagReport = lagReport;\n }\n\n // Once this change-streamer acquires \"ownership\" of the change DB,\n // it is safe to start the storer.\n await this.#storer.assumeOwnership();\n\n // The threshold in (estimated number of) bytes to send() on subscriber\n // websockets before `await`-ing the I/O buffers to be ready for more.\n const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n while (this.#state.shouldRun()) {\n let err: unknown;\n let watermark: string \| null = null;\n let unflushedBytes = 0;\n try {\n const {lastWatermark, backfillRequests} =\n await this.#storer.getStartStreamInitializationParameters();\n const stream = await this.#source.startStream(\n lastWatermark,\n backfillRequests,\n );\n this.#storer.run().catch(e => stream.changes.cancel(e));\n\n this.#stream = stream;\n this.#state.resetBackoff();\n this.#replicationStatusPublisher.publish(\n this.#lc,\n 'Replicating',\n `Replicating from ${lastWatermark}`,\n );\n watermark = null;\n\n for await (const change of stream.changes) {\n const [type, msg] = change;\n switch (type) {\n case 'status':\n if (msg.ack) {\n this.#storer.status(change); // storer acks once it gets through its queue\n }\n if (msg.lagReport) {\n // Lag reports are not stored in the cdc change log, but rather\n // only forwarded on \"live\" connections. When a new subscriber\n // is catching up, it is initialized with the #latestStatus\n // from which it can measure lag while catching up.\n this.#latestStatus.lagReport = msg.lagReport;\n this.#forwarder.sendStatus(this.#latestStatus);\n }\n continue;\n case 'control':\n await this.#handleControlMessage(msg);\n continue; // control messages are not stored/forwarded\n case 'begin':\n watermark = change[2].commitWatermark;\n break;\n case 'commit':\n if (watermark !== change[2].watermark) {\n throw new UnrecoverableError(\n `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n );\n }\n this.#txCounter.add(1);\n break;\n default:\n if (watermark === null) {\n throw new UnrecoverableError(\n `${type} change (${msg.tag}) received before 'begin' message`,\n );\n }\n break;\n }\n\n const entry: WatermarkedChange = [watermark, change];\n unflushedBytes += this.#storer.store(entry);\n if (unflushedBytes < flushBytesThreshold) {\n // pipeline changes until flushBytesThreshold\n this.#forwarder.forward(entry);\n } else {\n // Wait for messages to clear socket buffers to ensure that they\n // make their way to subscribers. Without this `await`, the\n // messages end up being buffered in this process, which:\n // (1) results in memory pressure and increased GC activity\n // (2) prevents subscribers from processing the messages as they\n // arrive, instead getting them in a large batch after being\n // idle while they were queued (causing further delays).\n await this.#forwarder.forwardWithFlowControl(entry);\n unflushedBytes = 0;\n }\n\n if (type === 'commit' \|\| type === 'rollback') {\n watermark = null;\n }\n\n // Allow the storer to exert back pressure.\n const readyForMore = this.#storer.readyForMore();\n if (readyForMore) {\n await readyForMore;\n }\n }\n } catch (e) {\n err = e;\n } finally {\n this.#stream?.changes.cancel();\n this.#stream = undefined;\n }\n\n // When the change stream is interrupted, abort any pending transaction.\n if (watermark) {\n this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n this.#storer.abort();\n this.#forwarder.forward([watermark, ['rollback', {tag: 'rollback'}]]);\n }\n\n // Backoff and drain any pending entries in the storer before reconnecting.\n await Promise.all([\n this.#storer.stop(),\n this.#state.backoff(this.#lc, err),\n this.#state.retryDelay > 5000\n ? publishCriticalEvent(\n this.#lc,\n replicationStatusError(this.#lc, 'Replicating', err),\n )\n : promiseVoid,\n ]);\n }\n\n this.#forwarder.stopProgressMonitor();\n this.#lc.info?.('ChangeStreamer stopped');\n }\n\n async #handleControlMessage(msg: ChangeStreamControl[1]) {\n this.#lc.info?.('received control message', msg);\n const {tag} = msg;\n\n switch (tag) {\n case 'reset-required':\n await markResetRequired(this.#changeDB, this.#shard);\n await publishReplicationError(\n this.#lc,\n 'Replicating',\n msg.message ?? 'Resync required',\n msg.errorDetails,\n );\n if (this.#autoReset) {\n this.#lc.warn?.('shutting down for auto-reset');\n await this.stop(new AutoResetSignal());\n }\n break;\n default:\n unreachable(tag);\n }\n }\n\n subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n if (mode === 'serving') {\n this.#serving.resolve();\n }\n const downstream = Subscription.create<Downstream>({\n cleanup: () => this.#forwarder.remove(subscriber),\n });\n const subscriber = new Subscriber(\n protocolVersion,\n id,\n watermark,\n downstream,\n () => this.#latestStatus,\n );\n if (replicaVersion !== this.#replicaVersion) {\n this.#lc.warn?.(\n `rejecting subscriber at replica version ${replicaVersion}`,\n );\n subscriber.close(\n ErrorType.WrongReplicaVersion,\n `current replica version is ${\n this.#replicaVersion\n } (requested ${replicaVersion})`,\n );\n } else {\n this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n this.#forwarder.add(subscriber);\n this.#storer.catchup(subscriber, mode);\n\n if (initial) {\n this.scheduleCleanup(watermark);\n }\n }\n return Promise.resolve(downstream);\n }\n\n scheduleCleanup(watermark: string) {\n const origSize = this.#initialWatermarks.size;\n this.#initialWatermarks.add(watermark);\n\n if (origSize === 0) {\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n\n async getChangeLogState(): Promise<{\n replicaVersion: string;\n minWatermark: string;\n }> {\n const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n if (!minWatermark) {\n this.#lc.warn?.(\n `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n );\n }\n return {\n replicaVersion: this.#replicaVersion,\n minWatermark: minWatermark ?? this.#replicaVersion,\n };\n }\n\n /\n Makes a best effort to purge the change log. In the event of a database\n * error, exceptions will be logged and swallowed, so this method is safe\n * to run in a timeout.\n /\n async #purgeOldChanges(): Promise<void> {\n const initial = [...this.#initialWatermarks];\n if (initial.length === 0) {\n this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n return;\n }\n const current = [...this.#forwarder.getAcks()];\n if (current.length === 0) {\n // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n // Bail to be safe.\n this.#lc.warn?.('No subscribers to confirm cleanup');\n return;\n }\n try {\n const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n if (earliestCurrent < earliestInitial) {\n this.#lc.info?.(\n `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n );\n } else {\n this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n const start = performance.now();\n const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n );\n this.#initialWatermarks.delete(earliestInitial);\n }\n } catch (e) {\n this.#lc.warn?.(`error purging change log`, e);\n } finally {\n if (this.#initialWatermarks.size) {\n // If there are unpurged watermarks to check, schedule the next purge.\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n }\n\n async stop(err?: unknown) {\n this.#state.stop(this.#lc, err);\n this.#stream?.changes.cancel();\n await this.#storer.stop();\n }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n// `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS 3;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAuDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,iCACA,oCACA,eAAe,YACiB;AAEhC,OAAM,yBAAyB,IAAI,UAAU,MAAM;AACnD,OAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,aACD;CAED,MAAM,EAAC,mBAAkB;AACzB,QAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,iCACA,oCACA,aACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyJH,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,KAAa;CAU/C,WAAoB,UAAU;CAE9B,aAAsB,mBACpB,eACA,gBACA,mCACD;CAED;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,WACA,iCACA,oCACA,eAAe,YACf;AACA,OAAK,KAAK;AACV,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AACzD,QAAA,QAAc;AACd,QAAA,WAAiB;AACjB,QAAA,iBAAuB;AACvB,QAAA,SAAe;AACf,QAAA,SAAe,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,MAAA,QAAc,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;GAAG,CAAC,GACzE,QAAO,KAAK,KAAK,IAAI,EACrB,gCACD;AACD,QAAA,YAAkB,IAAI,UAAU,IAAI,EAClC,oCACD,CAAC;AACF,QAAA,6BAAmC;AACnC,QAAA,YAAkB;AAClB,QAAA,QAAc,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,aAAa;AAChE,QAAA,eAAqB,EAAC,KAAK,UAAS;;CAGtC,MAAM,MAAM;AACV,QAAA,GAAS,OAAO,yBAAyB;AAEzC,QAAA,UAAgB,sBAAsB;EAEtC,MAAM,YAAY,MAAM,MAAA,OAAa,kBAAkB;AACvD,MAAI,UACF,OAAA,aAAmB,YAAY;AAKjC,QAAM,MAAA,OAAa,iBAAiB;EAIpC,MAAM,sBAAsB,wBAAwB,MAAM;AAE1D,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;AACrB,OAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,MAAA,OAAa,wCAAwC;IAC7D,MAAM,SAAS,MAAM,MAAA,OAAa,YAChC,eACA,iBACD;AACD,UAAA,OAAa,KAAK,CAAC,OAAM,MAAK,OAAO,QAAQ,OAAO,EAAE,CAAC;AAEvD,UAAA,SAAe;AACf,UAAA,MAAY,cAAc;AAC1B,UAAA,2BAAiC,QAC/B,MAAA,IACA,eACA,oBAAoB,gBACrB;AACD,gBAAY;AAEZ,eAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;AACpB,aAAQ,MAAR;MACE,KAAK;AACH,WAAI,IAAI,IACN,OAAA,OAAa,OAAO,OAAO;AAE7B,WAAI,IAAI,WAAW;AAKjB,cAAA,aAAmB,YAAY,IAAI;AACnC,cAAA,UAAgB,WAAW,MAAA,aAAmB;;AAEhD;MACF,KAAK;AACH,aAAM,MAAA,qBAA2B,IAAI;AACrC;MACF,KAAK;AACH,mBAAY,OAAO,GAAG;AACtB;MACF,KAAK;AACH,WAAI,cAAc,OAAO,GAAG,UAC1B,OAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,YAC7E;AAEH,aAAA,UAAgB,IAAI,EAAE;AACtB;MACF;AACE,WAAI,cAAc,KAChB,OAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,mCAC5B;AAEH;;KAGJ,MAAM,QAA2B,CAAC,WAAW,OAAO;AACpD,uBAAkB,MAAA,OAAa,MAAM,MAAM;AAC3C,SAAI,iBAAiB,oBAEnB,OAAA,UAAgB,QAAQ,MAAM;UACzB;AAQL,YAAM,MAAA,UAAgB,uBAAuB,MAAM;AACnD,uBAAiB;;AAGnB,SAAI,SAAS,YAAY,SAAS,WAChC,aAAY;KAId,MAAM,eAAe,MAAA,OAAa,cAAc;AAChD,SAAI,aACF,OAAM;;YAGH,GAAG;AACV,UAAM;aACE;AACR,UAAA,QAAc,QAAQ,QAAQ;AAC9B,UAAA,SAAe,KAAA;;AAIjB,OAAI,WAAW;AACb,UAAA,GAAS,OAAO,oCAAoC,YAAY;AAChE,UAAA,OAAa,OAAO;AACpB,UAAA,UAAgB,QAAQ,CAAC,WAAW,CAAC,YAAY,EAAC,KAAK,YAAW,CAAC,CAAC,CAAC;;AAIvE,SAAM,QAAQ,IAAI;IAChB,MAAA,OAAa,MAAM;IACnB,MAAA,MAAY,QAAQ,MAAA,IAAU,IAAI;IAClC,MAAA,MAAY,aAAa,MACrB,qBACE,MAAA,IACA,uBAAuB,MAAA,IAAU,eAAe,IAAI,CACrD,GACD;IACL,CAAC;;AAGJ,QAAA,UAAgB,qBAAqB;AACrC,QAAA,GAAS,OAAO,yBAAyB;;CAG3C,OAAA,qBAA4B,KAA6B;AACvD,QAAA,GAAS,OAAO,4BAA4B,IAAI;EAChD,MAAM,EAAC,QAAO;AAEd,UAAQ,KAAR;GACE,KAAK;AACH,UAAM,kBAAkB,MAAA,UAAgB,MAAA,MAAY;AACpD,UAAM,wBACJ,MAAA,IACA,eACA,IAAI,WAAW,mBACf,IAAI,aACL;AACD,QAAI,MAAA,WAAiB;AACnB,WAAA,GAAS,OAAO,+BAA+B;AAC/C,WAAM,KAAK,KAAK,IAAI,iBAAiB,CAAC;;AAExC;GACF,QACE,aAAY,IAAI;;;CAItB,UAAU,KAAqD;EAC7D,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,MAAI,SAAS,UACX,OAAA,QAAc,SAAS;EAEzB,MAAM,aAAa,aAAa,OAAmB,EACjD,eAAe,MAAA,UAAgB,OAAO,WAAW,EAClD,CAAC;EACF,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,MAAA,aACP;AACD,MAAI,mBAAmB,MAAA,gBAAsB;AAC3C,SAAA,GAAS,OACP,2CAA2C,iBAC5C;AACD,cAAW,MACT,GACA,8BACE,MAAA,eACD,cAAc,eAAe,GAC/B;SACI;AACL,SAAA,GAAS,QAAQ,qBAAqB,WAAW,KAAK;AAEtD,SAAA,UAAgB,IAAI,WAAW;AAC/B,SAAA,OAAa,QAAQ,YAAY,KAAK;AAEtC,OAAI,QACF,MAAK,gBAAgB,UAAU;;AAGnC,SAAO,QAAQ,QAAQ,WAAW;;CAGpC,gBAAgB,WAAmB;EACjC,MAAM,WAAW,MAAA,kBAAwB;AACzC,QAAA,kBAAwB,IAAI,UAAU;AAEtC,MAAI,aAAa,EACf,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;CAI3E,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,MAAA,OAAa,2BAA2B;AACnE,MAAI,CAAC,aACH,OAAA,GAAS,OACP,+EACD;AAEH,SAAO;GACL,gBAAgB,MAAA;GAChB,cAAc,gBAAgB,MAAA;GAC/B;;;;;;;CAQH,OAAA,kBAAwC;EACtC,MAAM,UAAU,CAAC,GAAG,MAAA,kBAAwB;AAC5C,MAAI,QAAQ,WAAW,GAAG;AACxB,SAAA,GAAS,OAAO,6CAA6C;AAC7D;;EAEF,MAAM,UAAU,CAAC,GAAG,MAAA,UAAgB,SAAS,CAAC;AAC9C,MAAI,QAAQ,WAAW,GAAG;AAGxB,SAAA,GAAS,OAAO,oCAAoC;AACpD;;AAEF,MAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,QAAoC;GACpE,MAAM,kBAAkB,IAAI,GAAI,QAAoC;AACpE,OAAI,kBAAkB,gBACpB,OAAA,GAAS,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,GAC/E;QACI;AACL,UAAA,GAAS,OAAO,0BAA0B,gBAAgB,MAAM;IAChE,MAAM,QAAQ,YAAY,KAAK;IAC/B,MAAM,UAAU,MAAM,MAAA,OAAa,mBAAmB,gBAAgB;IACtE,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,UAAA,GAAS,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,MACjE;AACD,UAAA,kBAAwB,OAAO,gBAAgB;;WAE1C,GAAG;AACV,SAAA,GAAS,OAAO,4BAA4B,EAAE;YACtC;AACR,OAAI,MAAA,kBAAwB,KAE1B,OAAA,MAAY,iBAAiB,MAAA,iBAAuB,EAAE,iBAAiB;;;CAK7E,MAAM,KAAK,KAAe;AACxB,QAAA,MAAY,KAAK,MAAA,IAAU,IAAI;AAC/B,QAAA,QAAc,QAAQ,QAAQ;AAC9B,QAAM,MAAA,OAAa,MAAM;;;AAmB7B,IAAM,mBAAmB,6BAA6B"}

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts CHANGED Viewed

@@ -100,9 +100,26 @@ export type ChangeEntry = {
  */
 export declare const statusSchema: v.ObjectType<{
     tag: v.Type<"status">;
+    lagReport: v.Optional<{
+        lastTimings?: {
+            sendTimeMs: number;
+            commitTimeMs: number;
+            receiveTimeMs: number;
+        } | undefined;
+        nextSendTimeMs: number;
+    }>;
 }, undefined>;
+export type Status = v.Infer<typeof statusSchema>;
 export declare const statusMessageSchema: v.TupleType<[v.Type<"status">, v.ObjectType<{
     tag: v.Type<"status">;
+    lagReport: v.Optional<{
+        lastTimings?: {
+            sendTimeMs: number;
+            commitTimeMs: number;
+            receiveTimeMs: number;
+        } | undefined;
+        nextSendTimeMs: number;
+    }>;
 }, undefined>]>;
 /**
  * A StatusMessage will be immediately sent on a (v2+) subscription to
@@ -122,6 +139,14 @@ declare const errorSchema: v.TupleType<[v.Type<"error">, v.ObjectType<{
 }, undefined>]>;
 export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status">, v.ObjectType<{
     tag: v.Type<"status">;
+    lagReport: v.Optional<{
+        lastTimings?: {
+            sendTimeMs: number;
+            commitTimeMs: number;
+            receiveTimeMs: number;
+        } | undefined;
+        nextSendTimeMs: number;
+    }>;
 }, undefined>]>, v.UnionType<[v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
     json: v.Optional<"p" | "s">;

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,2CAA2C,CAAC;AAEtE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;~~AAChE~~,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;CAChE;~~AAqBD~~,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY~~;;aAEvB~~,CAAC;AAEH,eAAO,MAAM,mBAAmB;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,qBAAsB,SAAQ,cAAc,EAAE,OAAO;IACpE;;;;OAIG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzC,iBAAiB,IAAI,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ"}
1	+ {"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,2CAA2C,CAAC;AAEtE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAEhE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;CAChE;AA0BD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;;;;;;;aASvB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,eAAO,MAAM,mBAAmB;;;;;;;;;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,qBAAsB,SAAQ,cAAc,EAAE,OAAO;IACpE;;;;OAIG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzC,iBAAiB,IAAI,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ"}

package/out/zero-cache/src/services/change-streamer/change-streamer.js CHANGED Viewed

@@ -1,12 +1,19 @@
 import { valita_exports } from "../../../../shared/src/valita.js";
 import "../change-source/protocol/current/data.js";
+import { changeSourceTimingsSchema } from "../replicator/reporter/report-schema.js";
 import { changeStreamDataSchema } from "../change-source/protocol/current/downstream.js";
 //#region ../zero-cache/src/services/change-streamer/change-streamer.ts
 /**
 * The StatusMessage payload for now is empty, but can be extended to
 * include meta-level information in the future.
 */
-var statusSchema = valita_exports.object({ tag: valita_exports.literal("status") });
+var statusSchema = valita_exports.object({
+	tag: valita_exports.literal("status"),
+	lagReport: valita_exports.object({
+		lastTimings: changeSourceTimingsSchema.optional(),
+		nextSendTimeMs: valita_exports.number()
+	}).optional()
+});
 var statusMessageSchema = valita_exports.tuple([valita_exports.literal("status"), statusSchema]);
 var subscriptionErrorSchema = valita_exports.object({
 	type: valita_exports.number(),

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {type Change} from '../change-source/protocol/current/data.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\n\n/*\n The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n \n * Changes from the upstream source are immediately forwarded to\n * connected subscribers to minimize latency.\n \n * They are then stored in a separate DB to facilitate catchup\n * of connecting subscribers that are behind.\n \n * Acknowledgements are sent upstream after they are successfully\n * stored.\n \n Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n \n However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n \n The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n /\nexport interface ChangeStreamer {\n /\n Subscribes to changes based on the supplied subscriber `ctx`,\n * which indicates the watermark at which the subscriber is up to\n * date.\n /\n subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n// - Adds the \"status\" message which is initially used to signal that the\n// subscription is valid (i.e. starting at the requested watermark).\n// v3: Adds the \"taskID\" to the subscription context, and support for\n// the BackupMonitor-mediated \"/snapshot\" request.\n// v4: Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n// status request so that a subscriber can verify whether its replica,\n// whether it be restored or existing in a permanent volume, is compatible\n// with the change-streamer.\n// v5: v0.26\n// - Moves relation.keyColumns and relation.replicaIdentity to\n// relation.rowKey: { columns, type }.\n// - Adds `metadata` to `create-table` message\n// - Adds `tableMetadata` to `add-column` message\n// - Adds `table-update-metadata` message\n// v6: v0.26\n// - Adds support for `backfill` messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n /\n The supported change-streamer protocol version.\n /\n protocolVersion: number;\n\n /\n Task ID. This is used to link the request with a preceding snapshot\n * reservation.\n /\n taskID: string \| null; // TODO: Make required when v3 is min.\n\n /\n Subscriber id. This is only used for debugging.\n /\n id: string;\n\n /\n The ReplicatorMode of the subscriber. 'backup' indicates that the\n * subscriber is local to the `change-streamer` in the `replication-manager`,\n * while 'serving' indicates that user-facing requests depend on the subscriber.\n /\n mode: ReplicatorMode;\n\n /\n The ChangeStreamer will return an Error if the subscriber is\n * on a different replica version (i.e. the initial snapshot associated\n * with the replication slot).\n /\n replicaVersion: string;\n\n /\n The watermark up to which the subscriber is up to date.\n * Only changes after the watermark will be streamed.\n /\n watermark: string;\n\n /\n Whether this is the first subscription request made by the task,\n * i.e. indicating that the watermark comes from a restored replica\n * backup. The ChangeStreamer uses this to determine which changes\n * are safe to purge from the Storer.\n /\n initial: boolean;\n};\n\nexport type ChangeEntry = {\n change: Change;\n\n /\n Note that it is technically possible for multiple changes to have\n * the same watermark, but that of a commit is guaranteed to be final,\n * so subscribers should only store the watermark of commit changes.\n /\n watermark: string;\n};\n\n/\n The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n /\nexport const statusSchema = v.object({\n tag: v.literal('status'),\n});\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/\n A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n /\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n type: v.number(), // ErrorType\n message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n statusMessageSchema,\n changeStreamDataSchema,\n errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\n/\n A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n \n A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n /\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService extends ChangeStreamer, Service {\n /\n Notifies the change streamer of a watermark that has been backed up,\n * indicating that changes before the watermark can be purged if active\n * subscribers have progressed beyond the watermark.\n */\n scheduleCleanup(watermark: string): void;\n\n getChangeLogState(): Promise<{\n replicaVersion: string;\n minWatermark: string;\n }>;\n}\n"],"mappings":"~~;;;;;;;;AAoIA~~,IAAa,eAAe,eAAE,OAAO,~~EACnC,~~KAAK,eAAE,QAAQ,SAAS,~~EACzB~~,CAAC;~~AAEF~~,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,SAAS,EAAE,aAAa,CAAC;AAU/E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ,CAAC,UAAU;CAC/B,CAAC;AAIF,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,EAAE,wBAAwB,CAAC;AAE1E,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,YACD"}
1	+ {"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {type Change} from '../change-source/protocol/current/data.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\n\n/*\n The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n \n * Changes from the upstream source are immediately forwarded to\n * connected subscribers to minimize latency.\n \n * They are then stored in a separate DB to facilitate catchup\n * of connecting subscribers that are behind.\n \n * Acknowledgements are sent upstream after they are successfully\n * stored.\n \n Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n \n However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n \n The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n /\nexport interface ChangeStreamer {\n /\n Subscribes to changes based on the supplied subscriber `ctx`,\n * which indicates the watermark at which the subscriber is up to\n * date.\n /\n subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: v0.18\n// - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n// - Adds the \"status\" message which is initially used to signal that the\n// subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n// - Adds the \"taskID\" to the subscription context, and support for\n// the BackupMonitor-mediated \"/snapshot\" request.\n// v4: v0.25\n// - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n// status request so that a subscriber can verify whether its replica,\n// whether it be restored or existing in a permanent volume, is compatible\n// with the change-streamer.\n// v5: v0.26\n// - Moves relation.keyColumns and relation.replicaIdentity to\n// relation.rowKey: { columns, type }.\n// - Adds `metadata` to `create-table` message\n// - Adds `tableMetadata` to `add-column` message\n// - Adds `table-update-metadata` message\n// v6: v0.26\n// - Adds support for `backfill` messages\n// v6: v1.0.1 (backwards compatible, no version change)\n// - Adds lag reporting to status messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n /\n The supported change-streamer protocol version.\n /\n protocolVersion: number;\n\n /\n Task ID. This is used to link the request with a preceding snapshot\n * reservation.\n /\n taskID: string \| null; // TODO: Make required when v3 is min.\n\n /\n Subscriber id. This is only used for debugging.\n /\n id: string;\n\n /\n The ReplicatorMode of the subscriber. 'backup' indicates that the\n * subscriber is local to the `change-streamer` in the `replication-manager`,\n * while 'serving' indicates that user-facing requests depend on the subscriber.\n /\n mode: ReplicatorMode;\n\n /\n The ChangeStreamer will return an Error if the subscriber is\n * on a different replica version (i.e. the initial snapshot associated\n * with the replication slot).\n /\n replicaVersion: string;\n\n /\n The watermark up to which the subscriber is up to date.\n * Only changes after the watermark will be streamed.\n /\n watermark: string;\n\n /\n Whether this is the first subscription request made by the task,\n * i.e. indicating that the watermark comes from a restored replica\n * backup. The ChangeStreamer uses this to determine which changes\n * are safe to purge from the Storer.\n /\n initial: boolean;\n};\n\nexport type ChangeEntry = {\n change: Change;\n\n /\n Note that it is technically possible for multiple changes to have\n * the same watermark, but that of a commit is guaranteed to be final,\n * so subscribers should only store the watermark of commit changes.\n /\n watermark: string;\n};\n\n/\n The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n /\nexport const statusSchema = v.object({\n tag: v.literal('status'),\n\n lagReport: v\n .object({\n lastTimings: changeSourceTimingsSchema.optional(),\n nextSendTimeMs: v.number(),\n })\n .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/\n A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n /\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n type: v.number(), // ErrorType\n message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n statusMessageSchema,\n changeStreamDataSchema,\n errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\n/\n A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n \n A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n /\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService extends ChangeStreamer, Service {\n /\n Notifies the change streamer of a watermark that has been backed up,\n * indicating that changes before the watermark can be purged if active\n * subscribers have progressed beyond the watermark.\n */\n scheduleCleanup(watermark: string): void;\n\n getChangeLogState(): Promise<{\n replicaVersion: string;\n minWatermark: string;\n }>;\n}\n"],"mappings":";;;;;;;;;AA0IA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,SAAS;CAExB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,UAAU;EACjD,gBAAgB,eAAE,QAAQ;EAC3B,CAAC,CACD,UAAU;CACd,CAAC;AAIF,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,SAAS,EAAE,aAAa,CAAC;AAU/E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,QAAQ;CAChB,SAAS,eAAE,QAAQ,CAAC,UAAU;CAC/B,CAAC;AAIF,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,EAAE,wBAAwB,CAAC;AAE1E,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,YACD"}

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts CHANGED Viewed

@@ -1,5 +1,6 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { WatermarkedChange } from './change-streamer-service.ts';
+import type { Status } from './change-streamer.ts';
 import type { Subscriber } from './subscriber.ts';
 export type ProgressMonitorOptions = {
     flowControlConsensusPaddingSeconds: number;
@@ -26,6 +27,7 @@ export declare class Forwarder {
      * occasionally to avoid memory blowup.
      */
     forward(entry: WatermarkedChange): void;
+    sendStatus(status: Status): void;
     /**
      * The flow-control-aware equivalent of {@link forward()}, returning a
      * Promise that resolves when replication should continue.

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,sBAAsB,GAAG;IACnC,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF,qBAAa,SAAS;;gBAWlB,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAAgE;IAMxE,oBAAoB;IAsBpB,mBAAmB;IAInB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IAKhC;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB;IA0CrD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}
1	+ {"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,sBAAsB,CAAC;AACjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,sBAAsB,GAAG;IACnC,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF,qBAAa,SAAS;;gBAWlB,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAAgE;IAMxE,oBAAoB;IAsBpB,mBAAmB;IAInB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IAKhC,UAAU,CAAC,MAAM,EAAE,MAAM;IAMzB;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB;IA0CrD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}

package/out/zero-cache/src/services/change-streamer/forwarder.js CHANGED Viewed

@@ -53,6 +53,9 @@ var Forwarder = class {
 		Broadcast.withoutTracking(this.#active.values(), entry);
 		this.#updateActiveSubscribers(entry[1]);
 	}
+	sendStatus(status) {
+		for (const sub of this.#active.values()) sub.sendStatus(status);
+	}
 	/**
 	* The flow-control-aware equivalent of {@link forward()}, returning a
 	* Promise that resolves when replication should continue.

package/out/zero-cache/src/services/change-streamer/forwarder.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwarder.js","names":["#lc","#progressMonitorOptions","#active","#queued","#progressMonitor","#trackProgress","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport {Broadcast} from './broadcast.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n flowControlConsensusPaddingSeconds: number;\n};\n\nexport class Forwarder {\n readonly #lc: LogContext;\n readonly #progressMonitorOptions: ProgressMonitorOptions;\n readonly #active = new Set<Subscriber>();\n readonly #queued = new Set<Subscriber>();\n #inTransaction = false;\n\n #currentBroadcast: Broadcast \| undefined;\n #progressMonitor: NodeJS.Timeout \| undefined;\n\n constructor(\n lc: LogContext,\n opts: ProgressMonitorOptions = {flowControlConsensusPaddingSeconds: 1},\n ) {\n this.#lc = lc.withContext('component', 'progress-monitor');\n this.#progressMonitorOptions = opts;\n }\n\n startProgressMonitor() {\n clearInterval(this.#progressMonitor);\n this.#progressMonitor = setInterval(this.#trackProgress, 1000);\n }\n\n readonly #trackProgress = () => {\n const now = performance.now();\n for (const sub of this.#active) {\n sub.sampleProcessRate(now);\n }\n\n const {flowControlConsensusPaddingSeconds} = this.#progressMonitorOptions;\n // A negative number disables early flow control release.\n if (flowControlConsensusPaddingSeconds >= 0) {\n this.#currentBroadcast?.checkProgress(\n this.#lc,\n flowControlConsensusPaddingSeconds * 1000,\n now,\n );\n }\n };\n\n stopProgressMonitor() {\n clearInterval(this.#progressMonitor);\n }\n\n /*\n `add()` is called in lock step with `Storer.catchup()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n /\n add(sub: Subscriber) {\n if (this.#inTransaction) {\n this.#queued.add(sub);\n } else {\n this.#active.add(sub);\n }\n }\n\n remove(sub: Subscriber) {\n this.#active.delete(sub);\n this.#queued.delete(sub);\n sub.close();\n }\n\n /\n `forward()` is called in lockstep with `Storer.store()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n \n This version of forward is fire-and-forget, with no flow control. The\n * change-streamer should call and await {@link forwardWithFlowControl()}\n * occasionally to avoid memory blowup.\n /\n forward(entry: WatermarkedChange) {\n Broadcast.withoutTracking(this.#active.values(), entry);\n this.#updateActiveSubscribers(entry[1]);\n }\n\n /\n The flow-control-aware equivalent of {@link forward()}, returning a\n * Promise that resolves when replication should continue.\n */\n async forwardWithFlowControl(entry: WatermarkedChange) {\n const broadcast = new Broadcast(this.#active.values(), entry);\n this.#updateActiveSubscribers(entry[1]);\n\n // set for progress tracking\n this.#currentBroadcast = broadcast;\n\n await broadcast.done;\n\n // Technically #currentBroadcast may have changed, so only\n // unset if it if is still the same.\n if (this.#currentBroadcast === broadcast) {\n this.#currentBroadcast = undefined;\n }\n }\n\n #updateActiveSubscribers([type]: ChangeStreamData) {\n switch (type) {\n case 'begin':\n // While in a Transaction, all added subscribers are \"queued\" so that no\n // messages are forwarded to them. This state corresponds to being queued\n // for catchup in the Storer, which will retrieve historic changes\n // and call catchup() once the current transaction is committed.\n this.#inTransaction = true;\n break;\n case 'commit':\n case 'rollback':\n // Upon commit or rollback, all queued subscribers are transferred to\n // the active set. This means that they can receive messages starting\n // from the next transaction.\n //\n // Note that if catchup is still in progress (in the Storer), these messages\n // will be buffered in the backlog until catchup completes.\n this.#inTransaction = false;\n for (const sub of this.#queued.values()) {\n this.#active.add(sub);\n }\n this.#queued.clear();\n break;\n }\n }\n\n getAcks(): Set<string> {\n return new Set(\n joinIterables(\n wrapIterable(this.#active).map(s => s.acked),\n wrapIterable(this.#queued).map(s => s.acked),\n ),\n );\n }\n}\n"],"mappings":";;;~~AAWA~~,IAAa,YAAb,MAAuB;CACrB;CACA;CACA,0BAAmB,IAAI,KAAiB;CACxC,0BAAmB,IAAI,KAAiB;CACxC,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAAC,oCAAoC,GAAE,EACtE;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,mBAAmB;AAC1D,QAAA,yBAA+B;;CAGjC,uBAAuB;AACrB,gBAAc,MAAA,gBAAsB;AACpC,QAAA,kBAAwB,YAAY,MAAA,eAAqB,IAAK;;CAGhE,uBAAgC;EAC9B,MAAM,MAAM,YAAY,KAAK;AAC7B,OAAK,MAAM,OAAO,MAAA,OAChB,KAAI,kBAAkB,IAAI;EAG5B,MAAM,EAAC,uCAAsC,MAAA;AAE7C,MAAI,sCAAsC,EACxC,OAAA,kBAAwB,cACtB,MAAA,IACA,qCAAqC,KACrC,IACD;;CAIL,sBAAsB;AACpB,gBAAc,MAAA,gBAAsB;;;;;;;CAQtC,IAAI,KAAiB;AACnB,MAAI,MAAA,cACF,OAAA,OAAa,IAAI,IAAI;MAErB,OAAA,OAAa,IAAI,IAAI;;CAIzB,OAAO,KAAiB;AACtB,QAAA,OAAa,OAAO,IAAI;AACxB,QAAA,OAAa,OAAO,IAAI;AACxB,MAAI,OAAO;;;;;;;;;;;CAYb,QAAQ,OAA0B;AAChC,YAAU,gBAAgB,MAAA,OAAa,QAAQ,EAAE,MAAM;AACvD,QAAA,wBAA8B,MAAM,GAAG;;;;;;~~CAOzC~~,MAAM,uBAAuB,OAA0B;EACrD,MAAM,YAAY,IAAI,UAAU,MAAA,OAAa,QAAQ,EAAE,MAAM;AAC7D,QAAA,wBAA8B,MAAM,GAAG;AAGvC,QAAA,mBAAyB;AAEzB,QAAM,UAAU;AAIhB,MAAI,MAAA,qBAA2B,UAC7B,OAAA,mBAAyB,KAAA;;CAI7B,yBAAyB,CAAC,OAAyB;AACjD,UAAQ,MAAR;GACE,KAAK;AAKH,UAAA,gBAAsB;AACtB;GACF,KAAK;GACL,KAAK;AAOH,UAAA,gBAAsB;AACtB,SAAK,MAAM,OAAO,MAAA,OAAa,QAAQ,CACrC,OAAA,OAAa,IAAI,IAAI;AAEvB,UAAA,OAAa,OAAO;AACpB;;;CAIN,UAAuB;AACrB,SAAO,IAAI,IACT,cACE,aAAa,MAAA,OAAa,CAAC,KAAI,MAAK,EAAE,MAAM,EAC5C,aAAa,MAAA,OAAa,CAAC,KAAI,MAAK,EAAE,MAAM,CAC7C,CACF"}
1	+ {"version":3,"file":"forwarder.js","names":["#lc","#progressMonitorOptions","#active","#queued","#progressMonitor","#trackProgress","#currentBroadcast","#inTransaction","#updateActiveSubscribers"],"sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport {Broadcast} from './broadcast.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Status} from './change-streamer.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n flowControlConsensusPaddingSeconds: number;\n};\n\nexport class Forwarder {\n readonly #lc: LogContext;\n readonly #progressMonitorOptions: ProgressMonitorOptions;\n readonly #active = new Set<Subscriber>();\n readonly #queued = new Set<Subscriber>();\n #inTransaction = false;\n\n #currentBroadcast: Broadcast \| undefined;\n #progressMonitor: NodeJS.Timeout \| undefined;\n\n constructor(\n lc: LogContext,\n opts: ProgressMonitorOptions = {flowControlConsensusPaddingSeconds: 1},\n ) {\n this.#lc = lc.withContext('component', 'progress-monitor');\n this.#progressMonitorOptions = opts;\n }\n\n startProgressMonitor() {\n clearInterval(this.#progressMonitor);\n this.#progressMonitor = setInterval(this.#trackProgress, 1000);\n }\n\n readonly #trackProgress = () => {\n const now = performance.now();\n for (const sub of this.#active) {\n sub.sampleProcessRate(now);\n }\n\n const {flowControlConsensusPaddingSeconds} = this.#progressMonitorOptions;\n // A negative number disables early flow control release.\n if (flowControlConsensusPaddingSeconds >= 0) {\n this.#currentBroadcast?.checkProgress(\n this.#lc,\n flowControlConsensusPaddingSeconds * 1000,\n now,\n );\n }\n };\n\n stopProgressMonitor() {\n clearInterval(this.#progressMonitor);\n }\n\n /*\n `add()` is called in lock step with `Storer.catchup()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n /\n add(sub: Subscriber) {\n if (this.#inTransaction) {\n this.#queued.add(sub);\n } else {\n this.#active.add(sub);\n }\n }\n\n remove(sub: Subscriber) {\n this.#active.delete(sub);\n this.#queued.delete(sub);\n sub.close();\n }\n\n /\n `forward()` is called in lockstep with `Storer.store()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n \n This version of forward is fire-and-forget, with no flow control. The\n * change-streamer should call and await {@link forwardWithFlowControl()}\n * occasionally to avoid memory blowup.\n /\n forward(entry: WatermarkedChange) {\n Broadcast.withoutTracking(this.#active.values(), entry);\n this.#updateActiveSubscribers(entry[1]);\n }\n\n sendStatus(status: Status) {\n for (const sub of this.#active.values()) {\n sub.sendStatus(status);\n }\n }\n\n /\n The flow-control-aware equivalent of {@link forward()}, returning a\n * Promise that resolves when replication should continue.\n */\n async forwardWithFlowControl(entry: WatermarkedChange) {\n const broadcast = new Broadcast(this.#active.values(), entry);\n this.#updateActiveSubscribers(entry[1]);\n\n // set for progress tracking\n this.#currentBroadcast = broadcast;\n\n await broadcast.done;\n\n // Technically #currentBroadcast may have changed, so only\n // unset if it if is still the same.\n if (this.#currentBroadcast === broadcast) {\n this.#currentBroadcast = undefined;\n }\n }\n\n #updateActiveSubscribers([type]: ChangeStreamData) {\n switch (type) {\n case 'begin':\n // While in a Transaction, all added subscribers are \"queued\" so that no\n // messages are forwarded to them. This state corresponds to being queued\n // for catchup in the Storer, which will retrieve historic changes\n // and call catchup() once the current transaction is committed.\n this.#inTransaction = true;\n break;\n case 'commit':\n case 'rollback':\n // Upon commit or rollback, all queued subscribers are transferred to\n // the active set. This means that they can receive messages starting\n // from the next transaction.\n //\n // Note that if catchup is still in progress (in the Storer), these messages\n // will be buffered in the backlog until catchup completes.\n this.#inTransaction = false;\n for (const sub of this.#queued.values()) {\n this.#active.add(sub);\n }\n this.#queued.clear();\n break;\n }\n }\n\n getAcks(): Set<string> {\n return new Set(\n joinIterables(\n wrapIterable(this.#active).map(s => s.acked),\n wrapIterable(this.#queued).map(s => s.acked),\n ),\n );\n }\n}\n"],"mappings":";;;AAYA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA,0BAAmB,IAAI,KAAiB;CACxC,0BAAmB,IAAI,KAAiB;CACxC,iBAAiB;CAEjB;CACA;CAEA,YACE,IACA,OAA+B,EAAC,oCAAoC,GAAE,EACtE;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,mBAAmB;AAC1D,QAAA,yBAA+B;;CAGjC,uBAAuB;AACrB,gBAAc,MAAA,gBAAsB;AACpC,QAAA,kBAAwB,YAAY,MAAA,eAAqB,IAAK;;CAGhE,uBAAgC;EAC9B,MAAM,MAAM,YAAY,KAAK;AAC7B,OAAK,MAAM,OAAO,MAAA,OAChB,KAAI,kBAAkB,IAAI;EAG5B,MAAM,EAAC,uCAAsC,MAAA;AAE7C,MAAI,sCAAsC,EACxC,OAAA,kBAAwB,cACtB,MAAA,IACA,qCAAqC,KACrC,IACD;;CAIL,sBAAsB;AACpB,gBAAc,MAAA,gBAAsB;;;;;;;CAQtC,IAAI,KAAiB;AACnB,MAAI,MAAA,cACF,OAAA,OAAa,IAAI,IAAI;MAErB,OAAA,OAAa,IAAI,IAAI;;CAIzB,OAAO,KAAiB;AACtB,QAAA,OAAa,OAAO,IAAI;AACxB,QAAA,OAAa,OAAO,IAAI;AACxB,MAAI,OAAO;;;;;;;;;;;CAYb,QAAQ,OAA0B;AAChC,YAAU,gBAAgB,MAAA,OAAa,QAAQ,EAAE,MAAM;AACvD,QAAA,wBAA8B,MAAM,GAAG;;CAGzC,WAAW,QAAgB;AACzB,OAAK,MAAM,OAAO,MAAA,OAAa,QAAQ,CACrC,KAAI,WAAW,OAAO;;;;;;CAQ1B,MAAM,uBAAuB,OAA0B;EACrD,MAAM,YAAY,IAAI,UAAU,MAAA,OAAa,QAAQ,EAAE,MAAM;AAC7D,QAAA,wBAA8B,MAAM,GAAG;AAGvC,QAAA,mBAAyB;AAEzB,QAAM,UAAU;AAIhB,MAAI,MAAA,qBAA2B,UAC7B,OAAA,mBAAyB,KAAA;;CAI7B,yBAAyB,CAAC,OAAyB;AACjD,UAAQ,MAAR;GACE,KAAK;AAKH,UAAA,gBAAsB;AACtB;GACF,KAAK;GACL,KAAK;AAOH,UAAA,gBAAsB;AACtB,SAAK,MAAM,OAAO,MAAA,OAAa,QAAQ,CACrC,OAAA,OAAa,IAAI,IAAI;AAEvB,UAAA,OAAa,OAAO;AACpB;;;CAIN,UAAuB;AACrB,SAAO,IAAI,IACT,cACE,aAAa,MAAA,OAAa,CAAC,KAAI,MAAK,EAAE,MAAM,EAC5C,aAAa,MAAA,OAAa,CAAC,KAAI,MAAK,EAAE,MAAM,CAC7C,CACF"}

package/out/zero-cache/src/services/change-streamer/subscriber.d.ts CHANGED Viewed

@@ -2,7 +2,7 @@ import type { Enum } from '../../../../shared/src/enum.ts';
 import type { Subscription } from '../../types/subscription.ts';
 import type { ChangeStreamData } from '../change-source/protocol/current.ts';
 import type { WatermarkedChange } from './change-streamer-service.ts';
-import { type Downstream } from './change-streamer.ts';
+import { type Downstream, type Status } from './change-streamer.ts';
 import * as ErrorType from './error-type-enum.ts';
 type ErrorType = Enum<typeof ErrorType>;
 /**
@@ -15,10 +15,11 @@ type ErrorType = Enum<typeof ErrorType>;
 export declare class Subscriber {
     #private;
     readonly id: string;
-    constructor(protocolVersion: number, id: string, watermark: string, downstream: Subscription<Downstream>);
+    constructor(protocolVersion: number, id: string, watermark: string, downstream: Subscription<Downstream>, latestStatus: () => Status);
     get watermark(): string;
     get acked(): string;
     send(change: WatermarkedChange): Promise<void>;
+    sendStatus(status: Status): void;
     /** catchup() is called on ChangeEntries loaded from the store. */
     catchup(change: WatermarkedChange): Promise<void>;
     /**

package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"subscriber.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAGzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,sCAAsC,CAAC;AAC3E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;~~AACrD~~,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAElD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC;;;;;;GAMG;AACH,qBAAa,UAAU;;IAErB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;~~gBAOlB~~,eAAe,EAAE,MAAM,EACvB,EAAE,EAAE,MAAM,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,YAAY,CAAC,UAAU,CAAC;~~IAUtC~~,IAAI,SAAS,WAEZ;IAED,IAAI,KAAK,WAER;IAEK,IAAI,CAAC,MAAM,EAAE,iBAAiB;~~IAoBpC~~,kEAAkE;IAC5D,OAAO,CAAC,MAAM,EAAE,iBAAiB;IAKvC;;;OAGG;IACH,WAAW;IA0DX;;OAEG;IACH,IAAI,UAAU,WAEb;IAED;;;OAGG;IACH,IAAI,YAAY,WAEf;IAED;;;OAGG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,SAAK,GAAG,IAAI;IAQrD,QAAQ,IAAI;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAC;IAalD,eAAe,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAS3C,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO;IAIlB,KAAK,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM;CAS1C"}
1	+ {"version":3,"file":"subscriber.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAGzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,sCAAsC,CAAC;AAC3E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAC,KAAK,UAAU,EAAE,KAAK,MAAM,EAAC,MAAM,sBAAsB,CAAC;AAClE,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAElD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC;;;;;;GAMG;AACH,qBAAa,UAAU;;IAErB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAQlB,eAAe,EAAE,MAAM,EACvB,EAAE,EAAE,MAAM,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,YAAY,CAAC,UAAU,CAAC,EACpC,YAAY,EAAE,MAAM,MAAM;IAW5B,IAAI,SAAS,WAEZ;IAED,IAAI,KAAK,WAER;IAEK,IAAI,CAAC,MAAM,EAAE,iBAAiB;IAwBpC,UAAU,CAAC,MAAM,EAAE,MAAM;IAMzB,kEAAkE;IAC5D,OAAO,CAAC,MAAM,EAAE,iBAAiB;IAKvC;;;OAGG;IACH,WAAW;IA0DX;;OAEG;IACH,IAAI,UAAU,WAEb;IAED;;;OAGG;IACH,IAAI,YAAY,WAEf;IAED;;;OAGG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,SAAK,GAAG,IAAI;IAQrD,QAAQ,IAAI;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAC;IAalD,eAAe,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAS3C,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO;IAIlB,KAAK,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM;CAS1C"}

package/out/zero-cache/src/services/change-streamer/subscriber.js CHANGED Viewed

@@ -14,13 +14,15 @@ var Subscriber = class {
 	#protocolVersion;
 	id;
 	#downstream;
+	#latestStatus;
 	#watermark;
 	#acked;
 	#backlog;
-	constructor(protocolVersion, id, watermark, downstream) {
+	constructor(protocolVersion, id, watermark, downstream, latestStatus) {
 		this.#protocolVersion = protocolVersion;
 		this.id = id;
 		this.#downstream = downstream;
+		this.#latestStatus = latestStatus;
 		this.#watermark = watermark;
 		this.#acked = watermark;
 		this.#backlog = [];
@@ -36,16 +38,23 @@ var Subscriber = class {
 		if (watermark > this.#watermark) if (this.#backlog) this.#backlog.push(change);
 		else await this.#sendChange(change);
 	}
-	#initialStatusSent = false;
-	#ensureInitialStatusSent() {
-		if (this.#protocolVersion >= 2 && !this.#initialStatusSent) {
-			this.#sendDownstream(["status", { tag: "status" }]);
-			this.#initialStatusSent = true;
+	#initialized = false;
+	/**
+	* Called once the subscriber's watermark has been validated in the initial
+	* catchup process.
+	*/
+	#initialize() {
+		if (!this.#initialized) {
+			this.#initialized = true;
+			this.sendStatus(this.#latestStatus());
 		}
 	}
+	sendStatus(status) {
+		if (this.#protocolVersion >= 2 && this.#initialized) this.#sendDownstream(["status", status]);
+	}
 	/** catchup() is called on ChangeEntries loaded from the store. */
 	async catchup(change) {
-		this.#ensureInitialStatusSent();
+		this.#initialize();
 		await this.#sendChange(change);
 	}
 	/**
@@ -53,7 +62,7 @@ var Subscriber = class {
 	* entries that were received during the catchup.
 	*/
 	setCaughtUp() {
-		this.#ensureInitialStatusSent();
+		this.#initialize();
 		assert(this.#backlog, "setCaughtUp() called but subscriber is not in catchup mode");
 		for (const change of this.#backlog) this.#sendChange(change);
 		this.#backlog = null;

package/out/zero-cache/src/services/change-streamer/subscriber.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#watermark","#acked","#backlog","#sendChange","#~~initialStatusSent~~","#sendDownstream","#~~ensureInitialStatusSent~~","#pending","#processed","#samples"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type Downstream} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/*\n Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n /\nexport class Subscriber {\n readonly #protocolVersion: number;\n readonly id: string;\n readonly #downstream: Subscription<Downstream>;\n #watermark: string;\n #acked: string;\n #backlog: WatermarkedChange[] \| null;\n\n constructor(\n protocolVersion: number,\n id: string,\n watermark: string,\n downstream: Subscription<Downstream>,\n ) {\n this.#protocolVersion = protocolVersion;\n this.id = id;\n this.#downstream = downstream;\n this.#watermark = watermark;\n this.#acked = watermark;\n this.#backlog = [];\n }\n\n get watermark() {\n return this.#watermark;\n }\n\n get acked() {\n return this.#acked;\n }\n\n async send(change: WatermarkedChange) {\n const [watermark] = change;\n if (watermark > this.#watermark) {\n if (this.#backlog) {\n this.#backlog.push(change);\n } else {\n await this.#sendChange(change);\n }\n }\n }\n\n #~~initialStatusSent~~ = false;\n\n #~~ensureInitialStatusSent~~() {\n if (this.#protocolVersion >= 2 && !this.#~~initialStatusSent~~) {\n void this.#sendDownstream(['status', ~~{tag: '~~status'}]);\n ~~this.#initialStatusSent = true;\n~~ }\n }\n\n /* catchup() is called on ChangeEntries loaded from the store. /\n async catchup(change: WatermarkedChange) {\n this.#~~ensureInitialStatusSent~~();\n await this.#sendChange(change);\n }\n\n /\n Marks the Subscribe as \"caught up\" and flushes any backlog of\n * entries that were received during the catchup.\n /\n setCaughtUp() {\n this.#~~ensureInitialStatusSent~~();\n assert(\n this.#backlog,\n 'setCaughtUp() called but subscriber is not in catchup mode',\n );\n // Note that this method must be asynchronous in order for send() to\n // interpret the #backlog variable correctly. This is the only place\n // where I/O flow control is not heeded. However, it will be awaited\n // by the next caller to send().\n for (const change of this.#backlog) {\n void this.#sendChange(change);\n }\n this.#backlog = null;\n }\n\n async #sendChange(change: WatermarkedChange) {\n const [watermark, downstream] = change;\n if (watermark <= this.watermark) {\n return;\n }\n if (!this.supportsMessage(downstream[1])) {\n return;\n }\n if (downstream[0] === 'commit') {\n this.#watermark = watermark;\n }\n const result = await this.#sendDownstream(downstream);\n if (downstream[0] === 'commit' && result === 'consumed') {\n this.#acked = max(this.#acked, watermark);\n }\n }\n\n async #sendDownstream(downstream: Downstream) {\n this.#pending++;\n const {result} = this.#downstream.push(downstream);\n try {\n return await result;\n } finally {\n this.#pending--;\n this.#processed++;\n }\n }\n\n // `pending` and `processed` stats are tracked by periodically sampling\n // the running totals (by the progress tracker in the Forwarder).\n // This information was originally collected for use in flow control\n // decisions. The final flow control algorithm ended up being simpler\n // than expected and does not actually use this information. However, the\n // stats are still tracked and logged during flow control decisions for\n // debugging, forensics, and potential improvements to the algorithm.\n\n #pending = 0;\n #processed = 0;\n #samples: {processed: number; timestamp: number}[] = [\n {processed: 0, timestamp: performance.now()},\n ];\n\n /\n The number of downstream messages that have yet to be acked.\n /\n get numPending() {\n return this.#pending;\n }\n\n /\n The total number of downstream messages that the subscriber has\n * processed (i.e. acked).\n /\n get numProcessed() {\n return this.#processed;\n }\n\n /\n Records a new history entry for the number of messages processed,\n * keeping the number of samples bounded to `maxSamples`.\n */\n sampleProcessRate(now: number, maxSamples = 10): this {\n while (this.#samples.length >= maxSamples) {\n this.#samples.shift();\n }\n this.#samples.push({processed: this.#processed, timestamp: now});\n return this;\n }\n\n getStats(): {processRate: number; pending: number} {\n const pending = this.#pending;\n if (this.#samples.length < 2) {\n return {processRate: 0, pending};\n }\n const from = this.#samples[0];\n const to = must(this.#samples.at(-1));\n const processed = to.processed - from.processed;\n const seconds = (to.timestamp - from.timestamp) / 1000;\n const processRate = seconds === 0 ? 0 : processed / seconds;\n return {processRate, pending};\n }\n\n supportsMessage(change: ChangeStreamData[1]) {\n switch (change.tag) {\n case 'update-table-metadata':\n // update-table-row-key is only understood by subscribers >= protocol v5\n return this.#protocolVersion >= 5;\n }\n return true;\n }\n\n fail(err?: unknown) {\n this.close(ErrorType.Unknown, String(err));\n }\n\n close(error?: ErrorType, message?: string) {\n if (error) {\n const {result} = this.#downstream.push(['error', {type: error, message}]);\n // Wait for the ACK of the error message before closing the connection.\n void result.then(() => this.#downstream.cancel());\n } else {\n this.#downstream.cancel();\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;AAmBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,iBACA,IACA,WACA,YACA;AACA,QAAA,kBAAwB;AACxB,OAAK,KAAK;AACV,QAAA,aAAmB;AACnB,QAAA,YAAkB;AAClB,QAAA,QAAc;AACd,QAAA,UAAgB,EAAE;;CAGpB,IAAI,YAAY;AACd,SAAO,MAAA;;CAGT,IAAI,QAAQ;AACV,SAAO,MAAA;;CAGT,MAAM,KAAK,QAA2B;EACpC,MAAM,CAAC,aAAa;AACpB,MAAI,YAAY,MAAA,UACd,KAAI,MAAA,QACF,OAAA,QAAc,KAAK,OAAO;MAE1B,OAAM,MAAA,WAAiB,OAAO;;CAKpC,~~qBAAqB~~;~~CAErB~~,~~2BAA2B~~;AACzB,MAAI,MAAA,mBAAyB,KAAK,~~CAAC,~~MAAA,~~mBAAyB;AACrD~~,~~SAAA~~,eAAqB,CAAC,UAAU,~~EAAC~~,~~KAAK,UAAS,~~CAAC,~~CAAC;AACtD,SAAA,oBAA0B;;;;CAK9B,~~MAAM,QAAQ,QAA2B;AACvC,QAAA,~~yBAA+B~~;~~AAC/B~~,QAAM,MAAA,WAAiB,OAAO;;;;;;CAOhC,cAAc;AACZ,QAAA,~~yBAA+B~~;~~AAC/B~~,SACE,MAAA,SACA,6DACD;AAKD,OAAK,MAAM,UAAU,MAAA,QACd,OAAA,WAAiB,OAAO;AAE/B,QAAA,UAAgB;;CAGlB,OAAA,WAAkB,QAA2B;EAC3C,MAAM,CAAC,WAAW,cAAc;AAChC,MAAI,aAAa,KAAK,UACpB;AAEF,MAAI,CAAC,KAAK,gBAAgB,WAAW,GAAG,CACtC;AAEF,MAAI,WAAW,OAAO,SACpB,OAAA,YAAkB;EAEpB,MAAM,SAAS,MAAM,MAAA,eAAqB,WAAW;AACrD,MAAI,WAAW,OAAO,YAAY,WAAW,WAC3C,OAAA,QAAc,IAAI,MAAA,OAAa,UAAU;;CAI7C,OAAA,eAAsB,YAAwB;AAC5C,QAAA;EACA,MAAM,EAAC,WAAU,MAAA,WAAiB,KAAK,WAAW;AAClD,MAAI;AACF,UAAO,MAAM;YACL;AACR,SAAA;AACA,SAAA;;;CAYJ,WAAW;CACX,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,KAAK;EAAC,CAC7C;;;;CAKD,IAAI,aAAa;AACf,SAAO,MAAA;;;;;;CAOT,IAAI,eAAe;AACjB,SAAO,MAAA;;;;;;CAOT,kBAAkB,KAAa,aAAa,IAAU;AACpD,SAAO,MAAA,QAAc,UAAU,WAC7B,OAAA,QAAc,OAAO;AAEvB,QAAA,QAAc,KAAK;GAAC,WAAW,MAAA;GAAiB,WAAW;GAAI,CAAC;AAChE,SAAO;;CAGT,WAAmD;EACjD,MAAM,UAAU,MAAA;AAChB,MAAI,MAAA,QAAc,SAAS,EACzB,QAAO;GAAC,aAAa;GAAG;GAAQ;EAElC,MAAM,OAAO,MAAA,QAAc;EAC3B,MAAM,KAAK,KAAK,MAAA,QAAc,GAAG,GAAG,CAAC;EACrC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;AAElD,SAAO;GAAC,aADY,YAAY,IAAI,IAAI,YAAY;GAC/B;GAAQ;;CAG/B,gBAAgB,QAA6B;AAC3C,UAAQ,OAAO,KAAf;GACE,KAAK,wBAEH,QAAO,MAAA,mBAAyB;;AAEpC,SAAO;;CAGT,KAAK,KAAe;AAClB,OAAK,MAAM,GAAmB,OAAO,IAAI,CAAC;;CAG5C,MAAM,OAAmB,SAAkB;AACzC,MAAI,OAAO;GACT,MAAM,EAAC,WAAU,MAAA,WAAiB,KAAK,CAAC,SAAS;IAAC,MAAM;IAAO;IAAQ,CAAC,CAAC;AAEpE,UAAO,WAAW,MAAA,WAAiB,QAAQ,CAAC;QAEjD,OAAA,WAAiB,QAAQ"}
1	+ {"version":3,"file":"subscriber.js","names":["#protocolVersion","#downstream","#latestStatus","#watermark","#acked","#backlog","#sendChange","#initialized","#sendDownstream","#initialize","#pending","#processed","#samples"],"sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type Downstream, type Status} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/*\n Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n /\nexport class Subscriber {\n readonly #protocolVersion: number;\n readonly id: string;\n readonly #downstream: Subscription<Downstream>;\n readonly #latestStatus: () => Status;\n #watermark: string;\n #acked: string;\n #backlog: WatermarkedChange[] \| null;\n\n constructor(\n protocolVersion: number,\n id: string,\n watermark: string,\n downstream: Subscription<Downstream>,\n latestStatus: () => Status,\n ) {\n this.#protocolVersion = protocolVersion;\n this.id = id;\n this.#downstream = downstream;\n this.#latestStatus = latestStatus;\n this.#watermark = watermark;\n this.#acked = watermark;\n this.#backlog = [];\n }\n\n get watermark() {\n return this.#watermark;\n }\n\n get acked() {\n return this.#acked;\n }\n\n async send(change: WatermarkedChange) {\n const [watermark] = change;\n if (watermark > this.#watermark) {\n if (this.#backlog) {\n this.#backlog.push(change);\n } else {\n await this.#sendChange(change);\n }\n }\n }\n\n #initialized = false;\n\n /\n Called once the subscriber's watermark has been validated in the initial\n * catchup process.\n /\n #initialize() {\n if (!this.#initialized) {\n this.#initialized = true;\n this.sendStatus(this.#latestStatus());\n }\n }\n\n sendStatus(status: Status) {\n if (this.#protocolVersion >= 2 && this.#initialized) {\n void this.#sendDownstream(['status', status]);\n }\n }\n\n /* catchup() is called on ChangeEntries loaded from the store. /\n async catchup(change: WatermarkedChange) {\n this.#initialize();\n await this.#sendChange(change);\n }\n\n /\n Marks the Subscribe as \"caught up\" and flushes any backlog of\n * entries that were received during the catchup.\n /\n setCaughtUp() {\n this.#initialize();\n assert(\n this.#backlog,\n 'setCaughtUp() called but subscriber is not in catchup mode',\n );\n // Note that this method must be asynchronous in order for send() to\n // interpret the #backlog variable correctly. This is the only place\n // where I/O flow control is not heeded. However, it will be awaited\n // by the next caller to send().\n for (const change of this.#backlog) {\n void this.#sendChange(change);\n }\n this.#backlog = null;\n }\n\n async #sendChange(change: WatermarkedChange) {\n const [watermark, downstream] = change;\n if (watermark <= this.watermark) {\n return;\n }\n if (!this.supportsMessage(downstream[1])) {\n return;\n }\n if (downstream[0] === 'commit') {\n this.#watermark = watermark;\n }\n const result = await this.#sendDownstream(downstream);\n if (downstream[0] === 'commit' && result === 'consumed') {\n this.#acked = max(this.#acked, watermark);\n }\n }\n\n async #sendDownstream(downstream: Downstream) {\n this.#pending++;\n const {result} = this.#downstream.push(downstream);\n try {\n return await result;\n } finally {\n this.#pending--;\n this.#processed++;\n }\n }\n\n // `pending` and `processed` stats are tracked by periodically sampling\n // the running totals (by the progress tracker in the Forwarder).\n // This information was originally collected for use in flow control\n // decisions. The final flow control algorithm ended up being simpler\n // than expected and does not actually use this information. However, the\n // stats are still tracked and logged during flow control decisions for\n // debugging, forensics, and potential improvements to the algorithm.\n\n #pending = 0;\n #processed = 0;\n #samples: {processed: number; timestamp: number}[] = [\n {processed: 0, timestamp: performance.now()},\n ];\n\n /\n The number of downstream messages that have yet to be acked.\n /\n get numPending() {\n return this.#pending;\n }\n\n /\n The total number of downstream messages that the subscriber has\n * processed (i.e. acked).\n /\n get numProcessed() {\n return this.#processed;\n }\n\n /\n Records a new history entry for the number of messages processed,\n * keeping the number of samples bounded to `maxSamples`.\n */\n sampleProcessRate(now: number, maxSamples = 10): this {\n while (this.#samples.length >= maxSamples) {\n this.#samples.shift();\n }\n this.#samples.push({processed: this.#processed, timestamp: now});\n return this;\n }\n\n getStats(): {processRate: number; pending: number} {\n const pending = this.#pending;\n if (this.#samples.length < 2) {\n return {processRate: 0, pending};\n }\n const from = this.#samples[0];\n const to = must(this.#samples.at(-1));\n const processed = to.processed - from.processed;\n const seconds = (to.timestamp - from.timestamp) / 1000;\n const processRate = seconds === 0 ? 0 : processed / seconds;\n return {processRate, pending};\n }\n\n supportsMessage(change: ChangeStreamData[1]) {\n switch (change.tag) {\n case 'update-table-metadata':\n // update-table-row-key is only understood by subscribers >= protocol v5\n return this.#protocolVersion >= 5;\n }\n return true;\n }\n\n fail(err?: unknown) {\n this.close(ErrorType.Unknown, String(err));\n }\n\n close(error?: ErrorType, message?: string) {\n if (error) {\n const {result} = this.#downstream.push(['error', {type: error, message}]);\n // Wait for the ACK of the error message before closing the connection.\n void result.then(() => this.#downstream.cancel());\n } else {\n this.#downstream.cancel();\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;AAmBA,IAAa,aAAb,MAAwB;CACtB;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,iBACA,IACA,WACA,YACA,cACA;AACA,QAAA,kBAAwB;AACxB,OAAK,KAAK;AACV,QAAA,aAAmB;AACnB,QAAA,eAAqB;AACrB,QAAA,YAAkB;AAClB,QAAA,QAAc;AACd,QAAA,UAAgB,EAAE;;CAGpB,IAAI,YAAY;AACd,SAAO,MAAA;;CAGT,IAAI,QAAQ;AACV,SAAO,MAAA;;CAGT,MAAM,KAAK,QAA2B;EACpC,MAAM,CAAC,aAAa;AACpB,MAAI,YAAY,MAAA,UACd,KAAI,MAAA,QACF,OAAA,QAAc,KAAK,OAAO;MAE1B,OAAM,MAAA,WAAiB,OAAO;;CAKpC,eAAe;;;;;CAMf,cAAc;AACZ,MAAI,CAAC,MAAA,aAAmB;AACtB,SAAA,cAAoB;AACpB,QAAK,WAAW,MAAA,cAAoB,CAAC;;;CAIzC,WAAW,QAAgB;AACzB,MAAI,MAAA,mBAAyB,KAAK,MAAA,YAC3B,OAAA,eAAqB,CAAC,UAAU,OAAO,CAAC;;;CAKjD,MAAM,QAAQ,QAA2B;AACvC,QAAA,YAAkB;AAClB,QAAM,MAAA,WAAiB,OAAO;;;;;;CAOhC,cAAc;AACZ,QAAA,YAAkB;AAClB,SACE,MAAA,SACA,6DACD;AAKD,OAAK,MAAM,UAAU,MAAA,QACd,OAAA,WAAiB,OAAO;AAE/B,QAAA,UAAgB;;CAGlB,OAAA,WAAkB,QAA2B;EAC3C,MAAM,CAAC,WAAW,cAAc;AAChC,MAAI,aAAa,KAAK,UACpB;AAEF,MAAI,CAAC,KAAK,gBAAgB,WAAW,GAAG,CACtC;AAEF,MAAI,WAAW,OAAO,SACpB,OAAA,YAAkB;EAEpB,MAAM,SAAS,MAAM,MAAA,eAAqB,WAAW;AACrD,MAAI,WAAW,OAAO,YAAY,WAAW,WAC3C,OAAA,QAAc,IAAI,MAAA,OAAa,UAAU;;CAI7C,OAAA,eAAsB,YAAwB;AAC5C,QAAA;EACA,MAAM,EAAC,WAAU,MAAA,WAAiB,KAAK,WAAW;AAClD,MAAI;AACF,UAAO,MAAM;YACL;AACR,SAAA;AACA,SAAA;;;CAYJ,WAAW;CACX,aAAa;CACb,WAAqD,CACnD;EAAC,WAAW;EAAG,WAAW,YAAY,KAAK;EAAC,CAC7C;;;;CAKD,IAAI,aAAa;AACf,SAAO,MAAA;;;;;;CAOT,IAAI,eAAe;AACjB,SAAO,MAAA;;;;;;CAOT,kBAAkB,KAAa,aAAa,IAAU;AACpD,SAAO,MAAA,QAAc,UAAU,WAC7B,OAAA,QAAc,OAAO;AAEvB,QAAA,QAAc,KAAK;GAAC,WAAW,MAAA;GAAiB,WAAW;GAAI,CAAC;AAChE,SAAO;;CAGT,WAAmD;EACjD,MAAM,UAAU,MAAA;AAChB,MAAI,MAAA,QAAc,SAAS,EACzB,QAAO;GAAC,aAAa;GAAG;GAAQ;EAElC,MAAM,OAAO,MAAA,QAAc;EAC3B,MAAM,KAAK,KAAK,MAAA,QAAc,GAAG,GAAG,CAAC;EACrC,MAAM,YAAY,GAAG,YAAY,KAAK;EACtC,MAAM,WAAW,GAAG,YAAY,KAAK,aAAa;AAElD,SAAO;GAAC,aADY,YAAY,IAAI,IAAI,YAAY;GAC/B;GAAQ;;CAG/B,gBAAgB,QAA6B;AAC3C,UAAQ,OAAO,KAAf;GACE,KAAK,wBAEH,QAAO,MAAA,mBAAyB;;AAEpC,SAAO;;CAGT,KAAK,KAAe;AAClB,OAAK,MAAM,GAAmB,OAAO,IAAI,CAAC;;CAG5C,MAAM,OAAmB,SAAkB;AACzC,MAAI,OAAO;GACT,MAAM,EAAC,WAAU,MAAA,WAAiB,KAAK,CAAC,SAAS;IAAC,MAAM;IAAO;IAAQ,CAAC,CAAC;AAEpE,UAAO,WAAW,MAAA,WAAiB,QAAQ,CAAC;QAEjD,OAAA,WAAiB,QAAQ"}

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts CHANGED Viewed

@@ -13,8 +13,8 @@ import type { WriteWorkerClient } from './write-worker-client.ts';
  */
 export declare class IncrementalSyncer {
     #private;
-    constructor(taskID: string, id: string, changeStreamer: ChangeStreamer, statusDb: Database, worker: WriteWorkerClient, mode: ReplicatorMode, publishReplicationStatus: boolean);
-    run(lc: LogContext): Promise<void>;
+    constructor(lc: LogContext, taskID: string, id: string, changeStreamer: ChangeStreamer, statusDb: Database, worker: WriteWorkerClient, mode: ReplicatorMode, publishReplicationStatus: boolean);
+    run(): Promise<void>;
     subscribe(): Source<ReplicaState>;
     stop(lc: LogContext, err?: unknown): void;
 }

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAE3D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,EAEL,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;AAK/C,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;~~AAClE~~,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;~~gBAmB1B~~,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,wBAAwB,EAAE,OAAO;~~IAY7B~~,GAAG~~,CAAC,EAAE,EAAE,UAAU~~;~~IA0IxB~~,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAIjC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO;CAGnC"}
1	+ {"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAE3D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,EAEL,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;AAK/C,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;gBAqB1B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,wBAAwB,EAAE,OAAO;IAc7B,GAAG;IAsJT,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAIjC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO;CAGnC"}

package/out/zero-cache/src/services/replicator/incremental-sync.js CHANGED Viewed

@@ -3,6 +3,7 @@ import { getOrCreateCounter } from "../../observability/metrics.js";
 import { ReplicationStatusPublisher } from "./replication-status.js";
 import "../change-streamer/change-streamer.js";
 import { Notifier } from "./notifier.js";
+import { ReplicationReportRecorder } from "./reporter/recorder.js";
 //#region ../zero-cache/src/services/replicator/incremental-sync.ts
 /**
 * The {@link IncrementalSyncer} manages a logical replication stream from upstream,
@@ -12,6 +13,7 @@ import { Notifier } from "./notifier.js";
 * in a worker thread via the {@link WriteWorkerClient}.
 */
 var IncrementalSyncer = class {
+	#lc;
 	#taskID;
 	#id;
 	#changeStreamer;
@@ -20,9 +22,11 @@ var IncrementalSyncer = class {
 	#mode;
 	#publishReplicationStatus;
 	#notifier;
+	#reporter;
 	#state = new RunningState("IncrementalSyncer");
 	#replicationEvents = getOrCreateCounter("replication", "events", "Number of replication events processed");
-	constructor(taskID, id, changeStreamer, statusDb, worker, mode, publishReplicationStatus) {
+	constructor(lc, taskID, id, changeStreamer, statusDb, worker, mode, publishReplicationStatus) {
+		this.#lc = lc;
 		this.#taskID = taskID;
 		this.#id = id;
 		this.#changeStreamer = changeStreamer;
@@ -31,8 +35,10 @@ var IncrementalSyncer = class {
 		this.#mode = mode;
 		this.#publishReplicationStatus = publishReplicationStatus;
 		this.#notifier = new Notifier();
+		this.#reporter = new ReplicationReportRecorder(lc);
 	}
-	async run(lc) {
+	async run() {
+		const lc = this.#lc;
 		this.#worker.onError((err) => this.#state.stop(lc, err));
 		lc.info?.(`Starting IncrementalSyncer`);
 		const { watermark: initialWatermark } = await this.#worker.getSubscriptionState();
@@ -60,9 +66,18 @@ var IncrementalSyncer = class {
 				for await (const message of downstream) {
 					this.#replicationEvents.add(1);
 					switch (message[0]) {
-						case "status":
-							lc.debug?.(`Received initial status`, message[1]);
+						case "status": {
+							const { lagReport } = message[1];
+							if (lagReport) {
+								const report = { nextSendTimeMs: lagReport.nextSendTimeMs };
+								if (lagReport.lastTimings) report.lastTimings = {
+									...lagReport.lastTimings,
+									replicateTimeMs: Date.now()
+								};
+								this.#reporter.record(report);
+							}
 							break;
+						}
 						case "error":
 							this.stop(lc, message[1]);
 							break;

package/out/zero-cache/src/services/replicator/incremental-sync.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"incremental-sync.js","names":["#taskID","#id","#changeStreamer","#statusDb","#worker","#mode","#publishReplicationStatus","#notifier","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport {\n PROTOCOL_VERSION,\n type ChangeStreamer,\n type Downstream,\n} from '../change-streamer/change-streamer.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {RunningState} from '../running-state.ts';\nimport {Notifier} from './notifier.ts';\nimport {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/*\n The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n readonly #taskID: string;\n readonly #id: string;\n readonly #changeStreamer: ChangeStreamer;\n readonly #statusDb: Database;\n readonly #worker: WriteWorkerClient;\n readonly #mode: ReplicatorMode;\n readonly #publishReplicationStatus: boolean;\n readonly #notifier: Notifier;\n\n readonly #state = new RunningState('IncrementalSyncer');\n\n readonly #replicationEvents = getOrCreateCounter(\n 'replication',\n 'events',\n 'Number of replication events processed',\n );\n\n constructor(\n taskID: string,\n id: string,\n changeStreamer: ChangeStreamer,\n statusDb: Database,\n worker: WriteWorkerClient,\n mode: ReplicatorMode,\n publishReplicationStatus: boolean,\n ) {\n this.#taskID = taskID;\n this.#id = id;\n this.#changeStreamer = changeStreamer;\n this.#statusDb = statusDb;\n this.#worker = worker;\n this.#mode = mode;\n this.#publishReplicationStatus = publishReplicationStatus;\n this.#notifier = new Notifier();\n }\n\n async run(lc: LogContext) {\n this.#worker.onError(err => this.#state.stop(lc, err));\n lc.info?.(`Starting IncrementalSyncer`);\n const {watermark: initialWatermark} =\n await this.#worker.getSubscriptionState();\n\n // Notify any waiting subscribers that the replica is ready to be read.\n void this.#notifier.notifySubscribers();\n\n // Only the backup replicator publishes replication status events.\n const statusPublisher = this.#publishReplicationStatus\n ? new ReplicationStatusPublisher(this.#statusDb)\n : undefined;\n\n while (this.#state.shouldRun()) {\n const {replicaVersion, watermark} =\n await this.#worker.getSubscriptionState();\n\n let downstream: Source<Downstream> \| undefined;\n let unregister = () => {};\n let err: unknown \| undefined;\n\n try {\n downstream = await this.#changeStreamer.subscribe({\n protocolVersion: PROTOCOL_VERSION,\n taskID: this.#taskID,\n id: this.#id,\n mode: this.#mode,\n watermark,\n replicaVersion,\n initial: watermark === initialWatermark,\n });\n this.#state.resetBackoff();\n unregister = this.#state.cancelOnStop(downstream);\n statusPublisher?.publish(\n lc,\n 'Replicating',\n `Replicating from ${watermark}`,\n );\n\n let backfillStatus: DownloadStatus \| undefined;\n\n for await (const message of downstream) {\n this.#replicationEvents.add(1);\n switch (message[0]) {\n case 'status':\n // Used for checking if a replica can be caught up. Not\n // relevant here.\n lc.debug?.(`Received initial status`, message[1]);\n break;\n case 'error':\n // Unrecoverable error. Stop the service.\n this.stop(lc, message[1]);\n break;\n default: {\n const msg = message[1];\n if (msg.tag === 'backfill' && msg.status) {\n const {status} = msg;\n if (!backfillStatus) {\n // Start publishing the status every 3 seconds.\n backfillStatus = status;\n statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilling ${msg.relation.name} table`,\n 3000,\n () =>\n backfillStatus\n ? {\n downloadStatus: [\n {\n ...backfillStatus,\n table: msg.relation.name,\n columns: [\n ...msg.relation.rowKey.columns,\n ...msg.columns,\n ],\n },\n ],\n }\n : {},\n );\n }\n backfillStatus = status; // Update the current status\n }\n\n const result = await this.#worker.processMessage(\n message as ChangeStreamData,\n );\n\n this.#handleResult(lc, result, statusPublisher);\n if (result?.completedBackfill) {\n backfillStatus = undefined;\n }\n break;\n }\n }\n }\n this.#worker.abort();\n } catch (e) {\n err = e;\n this.#worker.abort();\n } finally {\n downstream?.cancel();\n unregister();\n statusPublisher?.stop();\n }\n await this.#state.backoff(lc, err);\n }\n lc.info?.('IncrementalSyncer stopped');\n }\n\n #handleResult(\n lc: LogContext,\n result: CommitResult \| null,\n statusPublisher: ReplicationStatusPublisher \| undefined,\n ) {\n if (!result) {\n return;\n }\n if (result.completedBackfill) {\n // Publish the final status\n const status = result.completedBackfill;\n statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilled ${status.table} table`,\n 0,\n () => ({downloadStatus: [status]}),\n );\n } else if (result.schemaUpdated) {\n statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n }\n if (result.watermark && result.changeLogUpdated) {\n void this.#notifier.notifySubscribers({state: 'version-ready'});\n }\n }\n\n subscribe(): Source<ReplicaState> {\n return this.#notifier.subscribe();\n }\n\n stop(lc: LogContext, err?: unknown) {\n this.#state.stop(lc, err);\n }\n}\n"],"mappings":";;;;;;;;;;;;;AAyBA,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,oBAAoB;CAEvD,qBAA8B,mBAC5B,eACA,UACA,yCACD;CAED,YACE,QACA,IACA,gBACA,UACA,QACA,MACA,0BACA;AACA,QAAA,SAAe;AACf,QAAA,KAAW;AACX,QAAA,iBAAuB;AACvB,QAAA,WAAiB;AACjB,QAAA,SAAe;AACf,QAAA,OAAa;AACb,QAAA,2BAAiC;AACjC,QAAA,WAAiB,IAAI,UAAU;;CAGjC,MAAM,IAAI,IAAgB;AACxB,QAAA,OAAa,SAAQ,QAAO,MAAA,MAAY,KAAK,IAAI,IAAI,CAAC;AACtD,KAAG,OAAO,6BAA6B;EACvC,MAAM,EAAC,WAAW,qBAChB,MAAM,MAAA,OAAa,sBAAsB;AAGtC,QAAA,SAAe,mBAAmB;EAGvC,MAAM,kBAAkB,MAAA,2BACpB,IAAI,2BAA2B,MAAA,SAAe,GAC9C,KAAA;AAEJ,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,MAAA,OAAa,sBAAsB;GAE3C,IAAI;GACJ,IAAI,mBAAmB;GACvB,IAAI;AAEJ,OAAI;AACF,iBAAa,MAAM,MAAA,eAAqB,UAAU;KAChD,iBAAA;KACA,QAAQ,MAAA;KACR,IAAI,MAAA;KACJ,MAAM,MAAA;KACN;KACA;KACA,SAAS,cAAc;KACxB,CAAC;AACF,UAAA,MAAY,cAAc;AAC1B,iBAAa,MAAA,MAAY,aAAa,WAAW;AACjD,qBAAiB,QACf,IACA,eACA,oBAAoB,YACrB;IAED,IAAI;AAEJ,eAAW,MAAM,WAAW,YAAY;AACtC,WAAA,kBAAwB,IAAI,EAAE;AAC9B,aAAQ,QAAQ,IAAhB;MACE,KAAK;AAGH,UAAG,QAAQ,2BAA2B,QAAQ,GAAG;AACjD;MACF,KAAK;AAEH,YAAK,KAAK,IAAI,QAAQ,GAAG;AACzB;MACF,SAAS;OACP,MAAM,MAAM,QAAQ;AACpB,WAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,WAAU;AACjB,YAAI,CAAC,gBAAgB;AAEnB,0BAAiB;AACjB,0BAAiB,QACf,IACA,eACA,eAAe,IAAI,SAAS,KAAK,SACjC,WAEE,iBACI,EACE,gBAAgB,CACd;UACE,GAAG;UACH,OAAO,IAAI,SAAS;UACpB,SAAS,CACP,GAAG,IAAI,SAAS,OAAO,SACvB,GAAG,IAAI,QACR;UACF,CACF,EACF,GACD,EAAE,CACT;;AAEH,yBAAiB;;OAGnB,MAAM,SAAS,MAAM,MAAA,OAAa,eAChC,QACD;AAED,aAAA,aAAmB,IAAI,QAAQ,gBAAgB;AAC/C,WAAI,QAAQ,kBACV,kBAAiB,KAAA;AAEnB;;;;AAIN,UAAA,OAAa,OAAO;YACb,GAAG;AACV,UAAM;AACN,UAAA,OAAa,OAAO;aACZ;AACR,gBAAY,QAAQ;AACpB,gBAAY;AACZ,qBAAiB,MAAM;;AAEzB,SAAM,MAAA,MAAY,QAAQ,IAAI,IAAI;;AAEpC,KAAG,OAAO,4BAA4B;;CAGxC,cACE,IACA,QACA,iBACA;AACA,MAAI,CAAC,OACH;AAEF,MAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;AACtB,oBAAiB,QACf,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,OAAO,EAAC,EAClC;aACQ,OAAO,cAChB,kBAAiB,QAAQ,IAAI,eAAe,iBAAiB;AAE/D,MAAI,OAAO,aAAa,OAAO,iBACxB,OAAA,SAAe,kBAAkB,EAAC,OAAO,iBAAgB,CAAC;;CAInE,YAAkC;AAChC,SAAO,MAAA,SAAe,WAAW;;CAGnC,KAAK,IAAgB,KAAe;AAClC,QAAA,MAAY,KAAK,IAAI,IAAI"}
1	+ {"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#statusDb","#worker","#mode","#publishReplicationStatus","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n PROTOCOL_VERSION,\n type ChangeStreamer,\n type Downstream,\n} from '../change-streamer/change-streamer.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/*\n The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n readonly #lc: LogContext;\n readonly #taskID: string;\n readonly #id: string;\n readonly #changeStreamer: ChangeStreamer;\n readonly #statusDb: Database;\n readonly #worker: WriteWorkerClient;\n readonly #mode: ReplicatorMode;\n readonly #publishReplicationStatus: boolean;\n readonly #notifier: Notifier;\n readonly #reporter: ReplicationReportRecorder;\n\n readonly #state = new RunningState('IncrementalSyncer');\n\n readonly #replicationEvents = getOrCreateCounter(\n 'replication',\n 'events',\n 'Number of replication events processed',\n );\n\n constructor(\n lc: LogContext,\n taskID: string,\n id: string,\n changeStreamer: ChangeStreamer,\n statusDb: Database,\n worker: WriteWorkerClient,\n mode: ReplicatorMode,\n publishReplicationStatus: boolean,\n ) {\n this.#lc = lc;\n this.#taskID = taskID;\n this.#id = id;\n this.#changeStreamer = changeStreamer;\n this.#statusDb = statusDb;\n this.#worker = worker;\n this.#mode = mode;\n this.#publishReplicationStatus = publishReplicationStatus;\n this.#notifier = new Notifier();\n this.#reporter = new ReplicationReportRecorder(lc);\n }\n\n async run() {\n const lc = this.#lc;\n this.#worker.onError(err => this.#state.stop(lc, err));\n lc.info?.(`Starting IncrementalSyncer`);\n const {watermark: initialWatermark} =\n await this.#worker.getSubscriptionState();\n\n // Notify any waiting subscribers that the replica is ready to be read.\n void this.#notifier.notifySubscribers();\n\n // Only the backup replicator publishes replication status events.\n const statusPublisher = this.#publishReplicationStatus\n ? new ReplicationStatusPublisher(this.#statusDb)\n : undefined;\n\n while (this.#state.shouldRun()) {\n const {replicaVersion, watermark} =\n await this.#worker.getSubscriptionState();\n\n let downstream: Source<Downstream> \| undefined;\n let unregister = () => {};\n let err: unknown \| undefined;\n\n try {\n downstream = await this.#changeStreamer.subscribe({\n protocolVersion: PROTOCOL_VERSION,\n taskID: this.#taskID,\n id: this.#id,\n mode: this.#mode,\n watermark,\n replicaVersion,\n initial: watermark === initialWatermark,\n });\n this.#state.resetBackoff();\n unregister = this.#state.cancelOnStop(downstream);\n statusPublisher?.publish(\n lc,\n 'Replicating',\n `Replicating from ${watermark}`,\n );\n\n let backfillStatus: DownloadStatus \| undefined;\n\n for await (const message of downstream) {\n this.#replicationEvents.add(1);\n switch (message[0]) {\n case 'status': {\n const {lagReport} = message[1];\n if (lagReport) {\n const report: ReplicationReport = {\n nextSendTimeMs: lagReport.nextSendTimeMs,\n };\n if (lagReport.lastTimings) {\n report.lastTimings = {\n ...lagReport.lastTimings,\n replicateTimeMs: Date.now(),\n };\n }\n this.#reporter.record(report);\n }\n break;\n }\n case 'error':\n // Unrecoverable error. Stop the service.\n this.stop(lc, message[1]);\n break;\n default: {\n const msg = message[1];\n if (msg.tag === 'backfill' && msg.status) {\n const {status} = msg;\n if (!backfillStatus) {\n // Start publishing the status every 3 seconds.\n backfillStatus = status;\n statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilling ${msg.relation.name} table`,\n 3000,\n () =>\n backfillStatus\n ? {\n downloadStatus: [\n {\n ...backfillStatus,\n table: msg.relation.name,\n columns: [\n ...msg.relation.rowKey.columns,\n ...msg.columns,\n ],\n },\n ],\n }\n : {},\n );\n }\n backfillStatus = status; // Update the current status\n }\n\n const result = await this.#worker.processMessage(\n message as ChangeStreamData,\n );\n\n this.#handleResult(lc, result, statusPublisher);\n if (result?.completedBackfill) {\n backfillStatus = undefined;\n }\n break;\n }\n }\n }\n this.#worker.abort();\n } catch (e) {\n err = e;\n this.#worker.abort();\n } finally {\n downstream?.cancel();\n unregister();\n statusPublisher?.stop();\n }\n await this.#state.backoff(lc, err);\n }\n lc.info?.('IncrementalSyncer stopped');\n }\n\n #handleResult(\n lc: LogContext,\n result: CommitResult \| null,\n statusPublisher: ReplicationStatusPublisher \| undefined,\n ) {\n if (!result) {\n return;\n }\n if (result.completedBackfill) {\n // Publish the final status\n const status = result.completedBackfill;\n statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilled ${status.table} table`,\n 0,\n () => ({downloadStatus: [status]}),\n );\n } else if (result.schemaUpdated) {\n statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n }\n if (result.watermark && result.changeLogUpdated) {\n void this.#notifier.notifySubscribers({state: 'version-ready'});\n }\n }\n\n subscribe(): Source<ReplicaState> {\n return this.#notifier.subscribe();\n }\n\n stop(lc: LogContext, err?: unknown) {\n this.#state.stop(lc, err);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;AA2BA,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,oBAAoB;CAEvD,qBAA8B,mBAC5B,eACA,UACA,yCACD;CAED,YACE,IACA,QACA,IACA,gBACA,UACA,QACA,MACA,0BACA;AACA,QAAA,KAAW;AACX,QAAA,SAAe;AACf,QAAA,KAAW;AACX,QAAA,iBAAuB;AACvB,QAAA,WAAiB;AACjB,QAAA,SAAe;AACf,QAAA,OAAa;AACb,QAAA,2BAAiC;AACjC,QAAA,WAAiB,IAAI,UAAU;AAC/B,QAAA,WAAiB,IAAI,0BAA0B,GAAG;;CAGpD,MAAM,MAAM;EACV,MAAM,KAAK,MAAA;AACX,QAAA,OAAa,SAAQ,QAAO,MAAA,MAAY,KAAK,IAAI,IAAI,CAAC;AACtD,KAAG,OAAO,6BAA6B;EACvC,MAAM,EAAC,WAAW,qBAChB,MAAM,MAAA,OAAa,sBAAsB;AAGtC,QAAA,SAAe,mBAAmB;EAGvC,MAAM,kBAAkB,MAAA,2BACpB,IAAI,2BAA2B,MAAA,SAAe,GAC9C,KAAA;AAEJ,SAAO,MAAA,MAAY,WAAW,EAAE;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,MAAA,OAAa,sBAAsB;GAE3C,IAAI;GACJ,IAAI,mBAAmB;GACvB,IAAI;AAEJ,OAAI;AACF,iBAAa,MAAM,MAAA,eAAqB,UAAU;KAChD,iBAAA;KACA,QAAQ,MAAA;KACR,IAAI,MAAA;KACJ,MAAM,MAAA;KACN;KACA;KACA,SAAS,cAAc;KACxB,CAAC;AACF,UAAA,MAAY,cAAc;AAC1B,iBAAa,MAAA,MAAY,aAAa,WAAW;AACjD,qBAAiB,QACf,IACA,eACA,oBAAoB,YACrB;IAED,IAAI;AAEJ,eAAW,MAAM,WAAW,YAAY;AACtC,WAAA,kBAAwB,IAAI,EAAE;AAC9B,aAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;AAC5B,WAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,gBAC3B;AACD,YAAI,UAAU,YACZ,QAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,KAAK;SAC5B;AAEH,cAAA,SAAe,OAAO,OAAO;;AAE/B;;MAEF,KAAK;AAEH,YAAK,KAAK,IAAI,QAAQ,GAAG;AACzB;MACF,SAAS;OACP,MAAM,MAAM,QAAQ;AACpB,WAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,WAAU;AACjB,YAAI,CAAC,gBAAgB;AAEnB,0BAAiB;AACjB,0BAAiB,QACf,IACA,eACA,eAAe,IAAI,SAAS,KAAK,SACjC,WAEE,iBACI,EACE,gBAAgB,CACd;UACE,GAAG;UACH,OAAO,IAAI,SAAS;UACpB,SAAS,CACP,GAAG,IAAI,SAAS,OAAO,SACvB,GAAG,IAAI,QACR;UACF,CACF,EACF,GACD,EAAE,CACT;;AAEH,yBAAiB;;OAGnB,MAAM,SAAS,MAAM,MAAA,OAAa,eAChC,QACD;AAED,aAAA,aAAmB,IAAI,QAAQ,gBAAgB;AAC/C,WAAI,QAAQ,kBACV,kBAAiB,KAAA;AAEnB;;;;AAIN,UAAA,OAAa,OAAO;YACb,GAAG;AACV,UAAM;AACN,UAAA,OAAa,OAAO;aACZ;AACR,gBAAY,QAAQ;AACpB,gBAAY;AACZ,qBAAiB,MAAM;;AAEzB,SAAM,MAAA,MAAY,QAAQ,IAAI,IAAI;;AAEpC,KAAG,OAAO,4BAA4B;;CAGxC,cACE,IACA,QACA,iBACA;AACA,MAAI,CAAC,OACH;AAEF,MAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;AACtB,oBAAiB,QACf,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,OAAO,EAAC,EAClC;aACQ,OAAO,cAChB,kBAAiB,QAAQ,IAAI,eAAe,iBAAiB;AAE/D,MAAI,OAAO,aAAa,OAAO,iBACxB,OAAA,SAAe,kBAAkB,EAAC,OAAO,iBAAgB,CAAC;;CAInE,YAAkC;AAChC,SAAO,MAAA,SAAe,WAAW;;CAGnC,KAAK,IAAgB,KAAe;AAClC,QAAA,MAAY,KAAK,IAAI,IAAI"}