npm - @rocicorp/zero - Versions diffs - 0.26.0 → 0.26.1-canary.11 - Mend

@rocicorp/zero 0.26.0 → 0.26.1-canary.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (149) hide show

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer-service.js","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n min,\n type AtLeastOne,\n type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n ChangeSource,\n ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n type ChangeStreamControl,\n type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {publishReplicationError} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n DEFAULT_MAX_RETRY_DELAY_MS,\n RunningState,\n UnrecoverableError,\n} from '../running-state.ts';\nimport {\n type ChangeStreamerService,\n type Downstream,\n type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n AutoResetSignal,\n ensureReplicationConfig,\n markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/*\n Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n /\nexport async function initializeStreamer(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n changeSource: ChangeSource,\n subscriptionState: SubscriptionState,\n autoReset: boolean,\n backPressureLimitHeapProportion: number,\n setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n // Make sure the ChangeLog DB is set up.\n await initChangeStreamerSchema(lc, changeDB, shard);\n await ensureReplicationConfig(\n lc,\n changeDB,\n subscriptionState,\n shard,\n autoReset,\n );\n\n const {replicaVersion} = subscriptionState;\n return new ChangeStreamerImpl(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n changeSource,\n autoReset,\n backPressureLimitHeapProportion,\n setTimeoutFn,\n );\n}\n\n/\n Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n \n However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n /\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/\n Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n \n ### Subscriber Catchup\n \n Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n \n \n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * \| Historic changes in storage \| Pending (streamed) tx \| Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * Replication stream\n * > > > > > > > > >\n * ^ ---> required catchup ---> ^\n * Subscriber watermark Subscription begins\n * ```\n \n Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n \n Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n \n * If the streamer is in the middle of a pending Transaction, the\n * Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n * state, new changes are not forwarded to the Subscriber, and catchup\n * is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n * by the Storer, it begins catchup on the Subscriber (with a READONLY\n * snapshot so that it does not block subsequent storage operations).\n * This catchup is thus guaranteed to load the change log entries of\n * that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n * Subscriber from the \"queued\" to the \"active\" set of clients such that\n * the Subscriber begins receiving new changes, starting from the next\n * Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n * is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n * and the Subscriber henceforth forwards changes as they are received.\n \n In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n \n ### Watermarks and ordering\n \n The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n \n https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n \n > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n \n In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n \n A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n \n ```\n * LSN: 1 2 3 4 5 6 7 8 9 10\n * tx1: begin data data data commit\n * tx2: begin data data data commit\n * ```\n \n will arrive as:\n \n ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n \n Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n \n Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n \n * A `commit` message is assigned to a watermark corresponding to its LSN.\n * These are guaranteed to be in commit order by definition.\n \n * `begin` and `data` messages are assigned to the watermark of the\n * preceding `commit` (the previous transaction, or the replication\n * slot's starting LSN) plus 1. This guarantees that they will be sorted\n * after the previously commit transaction even if their LSNs came before it.\n * This is referred to as the `preCommitWatermark`.\n \n * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n * the position of the message within its transaction, with the `begin` message\n * starting at `0`. This guarantees that `begin` and `data` messages will be\n * fetched in the original ChangeSource order during catchup.\n \n `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n \n ### Cleanup\n \n As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n \n More concretely:\n \n * The `initial`, backup-derived watermark is the earliest to which cleanup\n * should ever happen.\n \n * However, it is possible for the replica backup to be ahead of a connected\n * subscriber; and if a network error causes that subscriber to retry from its\n * last watermark, the change streamer must support it.\n \n Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n /\nclass ChangeStreamerImpl implements ChangeStreamerService {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #replicaVersion: string;\n readonly #source: ChangeSource;\n readonly #storer: Storer;\n readonly #forwarder: Forwarder;\n\n readonly #autoReset: boolean;\n readonly #state: RunningState;\n readonly #initialWatermarks = new Set<string>();\n\n // Starting the (Postgres) ChangeStream results in killing the previous\n // Postgres subscriber, potentially creating a gap in which the old\n // change-streamer has shut down and the new change-streamer has not yet\n // been recognized as \"healthy\" (and thus does not get any requests).\n //\n // To minimize this gap, delay starting the ChangeStream until the first\n // request from a `serving` replicator, indicating that higher level\n // load-balancing / routing logic has begun routing requests to this task.\n readonly #serving = resolver();\n\n readonly #txCounter = getOrCreateCounter(\n 'replication',\n 'transactions',\n 'Count of replicated transactions',\n );\n\n #stream: ChangeStream \| undefined;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n replicaVersion: string,\n source: ChangeSource,\n autoReset: boolean,\n backPressureLimitHeapProportion: number,\n setTimeoutFn = setTimeout,\n ) {\n this.id = `change-streamer`;\n this.#lc = lc.withContext('component', 'change-streamer');\n this.#shard = shard;\n this.#changeDB = changeDB;\n this.#replicaVersion = replicaVersion;\n this.#source = source;\n this.#storer = new Storer(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n err => this.stop(err),\n backPressureLimitHeapProportion,\n );\n this.#forwarder = new Forwarder();\n this.#autoReset = autoReset;\n this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n }\n\n async run() {\n this.#lc.info?.('starting change stream');\n\n // Once this change-streamer acquires \"ownership\" of the change DB,\n // it is safe to start the storer.\n await this.#storer.assumeOwnership();\n\n // The threshold in (estimated number of) bytes to send() on subscriber\n // websockets before `await`-ing the I/O buffers to be ready for more.\n const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n while (this.#state.shouldRun()) {\n let err: unknown;\n let watermark: string \| null = null;\n let unflushedBytes = 0;\n try {\n const {lastWatermark, backfillRequests} =\n await this.#storer.getStartStreamInitializationParameters();\n const stream = await this.#source.startStream(\n lastWatermark,\n backfillRequests,\n );\n this.#storer.run().catch(e => stream.changes.cancel(e));\n\n this.#stream = stream;\n this.#state.resetBackoff();\n watermark = null;\n\n for await (const change of stream.changes) {\n const [type, msg] = change;\n switch (type) {\n case 'status':\n if (msg.ack) {\n this.#storer.status(change); // storer acks once it gets through its queue\n }\n continue;\n case 'control':\n await this.#handleControlMessage(msg);\n continue; // control messages are not stored/forwarded\n case 'begin':\n watermark = change[2].commitWatermark;\n break;\n case 'commit':\n if (watermark !== change[2].watermark) {\n throw new UnrecoverableError(\n `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n );\n }\n this.#txCounter.add(1);\n break;\n default:\n if (watermark === null) {\n throw new UnrecoverableError(\n `${type} change (${msg.tag}) received before 'begin' message`,\n );\n }\n break;\n }\n\n unflushedBytes += this.#storer.store([watermark, change]);\n const sent = this.#forwarder.forward([watermark, change]);\n if (unflushedBytes >= flushBytesThreshold) {\n // Wait for messages to clear socket buffers to ensure that they\n // make their way to subscribers. Without this `await`, the\n // messages end up being buffered in this process, which:\n // (1) results in memory pressure and increased GC activity\n // (2) prevents subscribers from processing the messages as they\n // arrive, instead getting them in a large batch after being\n // idle while they were queued (causing further delays).\n await sent;\n unflushedBytes = 0;\n }\n\n if (type === 'commit' \|\| type === 'rollback') {\n watermark = null;\n }\n\n // Allow the storer to exert back pressure.\n const readyForMore = this.#storer.readyForMore();\n if (readyForMore) {\n await readyForMore;\n }\n }\n } catch (e) {\n err = e;\n } finally {\n this.#stream?.changes.cancel();\n this.#stream = undefined;\n }\n\n // When the change stream is interrupted, abort any pending transaction.\n if (watermark) {\n this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n this.#storer.abort();\n await this.#forwarder.forward([\n watermark,\n ['rollback', {tag: 'rollback'}],\n ]);\n }\n\n // Backoff and drain any pending entries in the storer before reconnecting.\n await Promise.all([\n this.#storer.stop(),\n this.#state.backoff(this.#lc, err),\n ]);\n }\n this.#lc.info?.('ChangeStreamer stopped');\n }\n\n async #handleControlMessage(msg: ChangeStreamControl[1]) {\n this.#lc.info?.('received control message', msg);\n const {tag} = msg;\n\n switch (tag) {\n case 'reset-required':\n await markResetRequired(this.#changeDB, this.#shard);\n await publishReplicationError(\n this.#lc,\n 'Replicating',\n msg.message ?? 'Resync required',\n msg.errorDetails,\n );\n if (this.#autoReset) {\n this.#lc.warn?.('shutting down for auto-reset');\n await this.stop(new AutoResetSignal());\n }\n break;\n default:\n unreachable(tag);\n }\n }\n\n subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n if (mode === 'serving') {\n this.#serving.resolve();\n }\n const downstream = Subscription.create<Downstream>({\n cleanup: () => this.#forwarder.remove(subscriber),\n });\n const subscriber = new Subscriber(\n protocolVersion,\n id,\n watermark,\n downstream,\n );\n if (replicaVersion !== this.#replicaVersion) {\n this.#lc.warn?.(\n `rejecting subscriber at replica version ${replicaVersion}`,\n );\n subscriber.close(\n ErrorType.WrongReplicaVersion,\n `current replica version is ${\n this.#replicaVersion\n } (requested ${replicaVersion})`,\n );\n } else {\n this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n this.#forwarder.add(subscriber);\n this.#storer.catchup(subscriber, mode);\n\n if (initial) {\n this.scheduleCleanup(watermark);\n }\n }\n return Promise.resolve(downstream);\n }\n\n scheduleCleanup(watermark: string) {\n const origSize = this.#initialWatermarks.size;\n this.#initialWatermarks.add(watermark);\n\n if (origSize === 0) {\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n\n async getChangeLogState(): Promise<{\n replicaVersion: string;\n minWatermark: string;\n }> {\n const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n if (!minWatermark) {\n this.#lc.warn?.(\n `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n );\n }\n return {\n replicaVersion: this.#replicaVersion,\n minWatermark: minWatermark ?? this.#replicaVersion,\n };\n }\n\n /\n Makes a best effort to purge the change log. In the event of a database\n * error, exceptions will be logged and swallowed, so this method is safe\n * to run in a timeout.\n /\n async #purgeOldChanges(): Promise<void> {\n const initial = [...this.#initialWatermarks];\n if (initial.length === 0) {\n this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n return;\n }\n const current = [...this.#forwarder.getAcks()];\n if (current.length === 0) {\n // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n // Bail to be safe.\n this.#lc.warn?.('No subscribers to confirm cleanup');\n return;\n }\n try {\n const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n if (earliestCurrent < earliestInitial) {\n this.#lc.info?.(\n `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n );\n } else {\n this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n const start = performance.now();\n const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n );\n this.#initialWatermarks.delete(earliestInitial);\n }\n } catch (e) {\n this.#lc.warn?.(`error purging change log`, e);\n } finally {\n if (this.#initialWatermarks.size) {\n // If there are unpurged watermarks to check, schedule the next purge.\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n }\n\n async stop(err?: unknown) {\n this.#state.stop(this.#lc, err);\n this.#stream?.changes.cancel();\n await this.#storer.stop();\n }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n// `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS 3;\n"],"names":["ErrorType.WrongReplicaVersion"],"mappings":";;;;;;;;;;;;;;;;AAgDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,mBACA,WACA,iCACA,eAAe,YACiB;AAEhC,QAAM,yBAAyB,IAAI,UAAU,KAAK;AAClD,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,EAAC,mBAAkB;AACzB,SAAO,IAAI;AAAA,IACT;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AAwJA,MAAM,mBAAoD;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,yCAAyB,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUzB,WAAW,SAAA;AAAA,EAEX,aAAa;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF;AAAA,EAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,WACA,iCACA,eAAe,YACf;AACA,SAAK,KAAK;AACV,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AACxD,SAAK,SAAS;AACd,SAAK,YAAY;AACjB,SAAK,kBAAkB;AACvB,SAAK,UAAU;AACf,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,CAAA,aAAY,KAAK,SAAS,KAAK,KAAK,CAAC,UAAU,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;AAAA,MACxE,CAAA,QAAO,KAAK,KAAK,GAAG;AAAA,MACpB;AAAA,IAAA;AAEF,SAAK,aAAa,IAAI,UAAA;AACtB,SAAK,aAAa;AAClB,SAAK,SAAS,IAAI,aAAa,KAAK,IAAI,QAAW,YAAY;AAAA,EACjE;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,IAAI,OAAO,wBAAwB;AAIxC,UAAM,KAAK,QAAQ,gBAAA;AAInB,UAAM,sBAAsB,wBAAwB,KAAK;AAEzD,WAAO,KAAK,OAAO,aAAa;AAC9B,UAAI;AACJ,UAAI,YAA2B;AAC/B,UAAI,iBAAiB;AACrB,UAAI;AACF,cAAM,EAAC,eAAe,iBAAA,IACpB,MAAM,KAAK,QAAQ,uCAAA;AACrB,cAAM,SAAS,MAAM,KAAK,QAAQ;AAAA,UAChC;AAAA,UACA;AAAA,QAAA;AAEF,aAAK,QAAQ,MAAM,MAAM,OAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;AAEtD,aAAK,UAAU;AACf,aAAK,OAAO,aAAA;AACZ,oBAAY;AAEZ,yBAAiB,UAAU,OAAO,SAAS;AACzC,gBAAM,CAAC,MAAM,GAAG,IAAI;AACpB,kBAAQ,MAAA;AAAA,YACN,KAAK;AACH,kBAAI,IAAI,KAAK;AACX,qBAAK,QAAQ,OAAO,MAAM;AAAA,cAC5B;AACA;AAAA,YACF,KAAK;AACH,oBAAM,KAAK,sBAAsB,GAAG;AACpC;AAAA;AAAA,YACF,KAAK;AACH,0BAAY,OAAO,CAAC,EAAE;AACtB;AAAA,YACF,KAAK;AACH,kBAAI,cAAc,OAAO,CAAC,EAAE,WAAW;AACrC,sBAAM,IAAI;AAAA,kBACR,oBAAoB,OAAO,CAAC,EAAE,SAAS,qCAAqC,SAAS;AAAA,gBAAA;AAAA,cAEzF;AACA,mBAAK,WAAW,IAAI,CAAC;AACrB;AAAA,YACF;AACE,kBAAI,cAAc,MAAM;AACtB,sBAAM,IAAI;AAAA,kBACR,GAAG,IAAI,YAAY,IAAI,GAAG;AAAA,gBAAA;AAAA,cAE9B;AACA;AAAA,UAAA;AAGJ,4BAAkB,KAAK,QAAQ,MAAM,CAAC,WAAW,MAAM,CAAC;AACxD,gBAAM,OAAO,KAAK,WAAW,QAAQ,CAAC,WAAW,MAAM,CAAC;AACxD,cAAI,kBAAkB,qBAAqB;AAQzC,kBAAM;AACN,6BAAiB;AAAA,UACnB;AAEA,cAAI,SAAS,YAAY,SAAS,YAAY;AAC5C,wBAAY;AAAA,UACd;AAGA,gBAAM,eAAe,KAAK,QAAQ,aAAA;AAClC,cAAI,cAAc;AAChB,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR,UAAA;AACE,aAAK,SAAS,QAAQ,OAAA;AACtB,aAAK,UAAU;AAAA,MACjB;AAGA,UAAI,WAAW;AACb,aAAK,IAAI,OAAO,oCAAoC,SAAS,EAAE;AAC/D,aAAK,QAAQ,MAAA;AACb,cAAM,KAAK,WAAW,QAAQ;AAAA,UAC5B;AAAA,UACA,CAAC,YAAY,EAAC,KAAK,YAAW;AAAA,QAAA,CAC/B;AAAA,MACH;AAGA,YAAM,QAAQ,IAAI;AAAA,QAChB,KAAK,QAAQ,KAAA;AAAA,QACb,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AAAA,MAAA,CAClC;AAAA,IACH;AACA,SAAK,IAAI,OAAO,wBAAwB;AAAA,EAC1C;AAAA,EAEA,MAAM,sBAAsB,KAA6B;AACvD,SAAK,IAAI,OAAO,4BAA4B,GAAG;AAC/C,UAAM,EAAC,QAAO;AAEd,YAAQ,KAAA;AAAA,MACN,KAAK;AACH,cAAM,kBAAkB,KAAK,WAAW,KAAK,MAAM;AACnD,cAAM;AAAA,UACJ,KAAK;AAAA,UACL;AAAA,UACA,IAAI,WAAW;AAAA,UACf,IAAI;AAAA,QAAA;AAEN,YAAI,KAAK,YAAY;AACnB,eAAK,IAAI,OAAO,8BAA8B;AAC9C,gBAAM,KAAK,KAAK,IAAI,iBAAiB;AAAA,QACvC;AACA;AAAA,MACF;AACE,oBAAe;AAAA,IAAA;AAAA,EAErB;AAAA,EAEA,UAAU,KAAqD;AAC7D,UAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,QAAI,SAAS,WAAW;AACtB,WAAK,SAAS,QAAA;AAAA,IAChB;AACA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM,KAAK,WAAW,OAAO,UAAU;AAAA,IAAA,CACjD;AACD,UAAM,aAAa,IAAI;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,mBAAmB,KAAK,iBAAiB;AAC3C,WAAK,IAAI;AAAA,QACP,2CAA2C,cAAc;AAAA,MAAA;AAE3D,iBAAW;AAAA,QACTA;AAAAA,QACA,8BACE,KAAK,eACP,eAAe,cAAc;AAAA,MAAA;AAAA,IAEjC,OAAO;AACL,WAAK,IAAI,QAAQ,qBAAqB,WAAW,EAAE,EAAE;AAErD,WAAK,WAAW,IAAI,UAAU;AAC9B,WAAK,QAAQ,QAAQ,YAAY,IAAI;AAErC,UAAI,SAAS;AACX,aAAK,gBAAgB,SAAS;AAAA,MAChC;AAAA,IACF;AACA,WAAO,QAAQ,QAAQ,UAAU;AAAA,EACnC;AAAA,EAEA,gBAAgB,WAAmB;AACjC,UAAM,WAAW,KAAK,mBAAmB;AACzC,SAAK,mBAAmB,IAAI,SAAS;AAErC,QAAI,aAAa,GAAG;AAClB,WAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,MAAM,oBAGH;AACD,UAAM,eAAe,MAAM,KAAK,QAAQ,0BAAA;AACxC,QAAI,CAAC,cAAc;AACjB,WAAK,IAAI;AAAA,QACP;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO;AAAA,MACL,gBAAgB,KAAK;AAAA,MACrB,cAAc,gBAAgB,KAAK;AAAA,IAAA;AAAA,EAEvC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,mBAAkC;AACtC,UAAM,UAAU,CAAC,GAAG,KAAK,kBAAkB;AAC3C,QAAI,QAAQ,WAAW,GAAG;AACxB,WAAK,IAAI,OAAO,4CAA4C;AAC5D;AAAA,IACF;AACA,UAAM,UAAU,CAAC,GAAG,KAAK,WAAW,SAAS;AAC7C,QAAI,QAAQ,WAAW,GAAG;AAGxB,WAAK,IAAI,OAAO,mCAAmC;AACnD;AAAA,IACF;AACA,QAAI;AACF,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,UAAI,kBAAkB,iBAAiB;AACrC,aAAK,IAAI;AAAA,UACP,yCAAyC,eAAe,MAAM,eAAe;AAAA,QAAA;AAAA,MAEjF,OAAO;AACL,aAAK,IAAI,OAAO,0BAA0B,eAAe,MAAM;AAC/D,cAAM,QAAQ,YAAY,IAAA;AAC1B,cAAM,UAAU,MAAM,KAAK,QAAQ,mBAAmB,eAAe;AACrE,cAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,aAAK,IAAI;AAAA,UACP,UAAU,OAAO,mBAAmB,eAAe,KAAK,OAAO;AAAA,QAAA;AAEjE,aAAK,mBAAmB,OAAO,eAAe;AAAA,MAChD;AAAA,IACF,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,4BAA4B,CAAC;AAAA,IAC/C,UAAA;AACE,UAAI,KAAK,mBAAmB,MAAM;AAEhC,aAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,KAAe;AACxB,SAAK,OAAO,KAAK,KAAK,KAAK,GAAG;AAC9B,SAAK,SAAS,QAAQ,OAAA;AACtB,UAAM,KAAK,QAAQ,KAAA;AAAA,EACrB;AACF;AAiBA,MAAM,mBAAmB,6BAA6B;"}
1	+ {"version":3,"file":"change-streamer-service.js","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {getDefaultHighWaterMark} from 'node:stream';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n min,\n type AtLeastOne,\n type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n ChangeSource,\n ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n type ChangeStreamControl,\n type ChangeStreamData,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n publishReplicationError,\n replicationStatusError,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n DEFAULT_MAX_RETRY_DELAY_MS,\n RunningState,\n UnrecoverableError,\n} from '../running-state.ts';\nimport {\n type ChangeStreamerService,\n type Downstream,\n type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n AutoResetSignal,\n ensureReplicationConfig,\n markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/*\n Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n /\nexport async function initializeStreamer(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n changeSource: ChangeSource,\n subscriptionState: SubscriptionState,\n autoReset: boolean,\n backPressureLimitHeapProportion: number,\n flowControlConsensusPaddingSeconds: number,\n setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n // Make sure the ChangeLog DB is set up.\n await initChangeStreamerSchema(lc, changeDB, shard);\n await ensureReplicationConfig(\n lc,\n changeDB,\n subscriptionState,\n shard,\n autoReset,\n );\n\n const {replicaVersion} = subscriptionState;\n return new ChangeStreamerImpl(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n changeSource,\n autoReset,\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n setTimeoutFn,\n );\n}\n\n/\n Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n \n However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n /\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\n/\n Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n \n ### Subscriber Catchup\n \n Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n \n \n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * \| Historic changes in storage \| Pending (streamed) tx \| Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * Replication stream\n * > > > > > > > > >\n * ^ ---> required catchup ---> ^\n * Subscriber watermark Subscription begins\n * ```\n \n Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n \n Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n \n * If the streamer is in the middle of a pending Transaction, the\n * Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n * state, new changes are not forwarded to the Subscriber, and catchup\n * is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n * by the Storer, it begins catchup on the Subscriber (with a READONLY\n * snapshot so that it does not block subsequent storage operations).\n * This catchup is thus guaranteed to load the change log entries of\n * that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n * Subscriber from the \"queued\" to the \"active\" set of clients such that\n * the Subscriber begins receiving new changes, starting from the next\n * Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n * is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n * and the Subscriber henceforth forwards changes as they are received.\n \n In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n \n ### Watermarks and ordering\n \n The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n \n https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n \n > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n \n In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n \n A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n \n ```\n * LSN: 1 2 3 4 5 6 7 8 9 10\n * tx1: begin data data data commit\n * tx2: begin data data data commit\n * ```\n \n will arrive as:\n \n ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n \n Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n \n Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n \n * A `commit` message is assigned to a watermark corresponding to its LSN.\n * These are guaranteed to be in commit order by definition.\n \n * `begin` and `data` messages are assigned to the watermark of the\n * preceding `commit` (the previous transaction, or the replication\n * slot's starting LSN) plus 1. This guarantees that they will be sorted\n * after the previously commit transaction even if their LSNs came before it.\n * This is referred to as the `preCommitWatermark`.\n \n * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n * the position of the message within its transaction, with the `begin` message\n * starting at `0`. This guarantees that `begin` and `data` messages will be\n * fetched in the original ChangeSource order during catchup.\n \n `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n \n ### Cleanup\n \n As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n \n More concretely:\n \n * The `initial`, backup-derived watermark is the earliest to which cleanup\n * should ever happen.\n \n * However, it is possible for the replica backup to be ahead of a connected\n * subscriber; and if a network error causes that subscriber to retry from its\n * last watermark, the change streamer must support it.\n \n Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n /\nclass ChangeStreamerImpl implements ChangeStreamerService {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #replicaVersion: string;\n readonly #source: ChangeSource;\n readonly #storer: Storer;\n readonly #forwarder: Forwarder;\n\n readonly #autoReset: boolean;\n readonly #state: RunningState;\n readonly #initialWatermarks = new Set<string>();\n\n // Starting the (Postgres) ChangeStream results in killing the previous\n // Postgres subscriber, potentially creating a gap in which the old\n // change-streamer has shut down and the new change-streamer has not yet\n // been recognized as \"healthy\" (and thus does not get any requests).\n //\n // To minimize this gap, delay starting the ChangeStream until the first\n // request from a `serving` replicator, indicating that higher level\n // load-balancing / routing logic has begun routing requests to this task.\n readonly #serving = resolver();\n\n readonly #txCounter = getOrCreateCounter(\n 'replication',\n 'transactions',\n 'Count of replicated transactions',\n );\n\n #stream: ChangeStream \| undefined;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n replicaVersion: string,\n source: ChangeSource,\n autoReset: boolean,\n backPressureLimitHeapProportion: number,\n flowControlConsensusPaddingSeconds: number,\n setTimeoutFn = setTimeout,\n ) {\n this.id = `change-streamer`;\n this.#lc = lc.withContext('component', 'change-streamer');\n this.#shard = shard;\n this.#changeDB = changeDB;\n this.#replicaVersion = replicaVersion;\n this.#source = source;\n this.#storer = new Storer(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n err => this.stop(err),\n backPressureLimitHeapProportion,\n );\n this.#forwarder = new Forwarder(lc, {\n flowControlConsensusPaddingSeconds,\n });\n this.#autoReset = autoReset;\n this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n }\n\n async run() {\n this.#lc.info?.('starting change stream');\n\n this.#forwarder.startProgressMonitor();\n\n // Once this change-streamer acquires \"ownership\" of the change DB,\n // it is safe to start the storer.\n await this.#storer.assumeOwnership();\n\n // The threshold in (estimated number of) bytes to send() on subscriber\n // websockets before `await`-ing the I/O buffers to be ready for more.\n const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n while (this.#state.shouldRun()) {\n let err: unknown;\n let watermark: string \| null = null;\n let unflushedBytes = 0;\n try {\n const {lastWatermark, backfillRequests} =\n await this.#storer.getStartStreamInitializationParameters();\n const stream = await this.#source.startStream(\n lastWatermark,\n backfillRequests,\n );\n this.#storer.run().catch(e => stream.changes.cancel(e));\n\n this.#stream = stream;\n this.#state.resetBackoff();\n watermark = null;\n\n for await (const change of stream.changes) {\n const [type, msg] = change;\n switch (type) {\n case 'status':\n if (msg.ack) {\n this.#storer.status(change); // storer acks once it gets through its queue\n }\n continue;\n case 'control':\n await this.#handleControlMessage(msg);\n continue; // control messages are not stored/forwarded\n case 'begin':\n watermark = change[2].commitWatermark;\n break;\n case 'commit':\n if (watermark !== change[2].watermark) {\n throw new UnrecoverableError(\n `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n );\n }\n this.#txCounter.add(1);\n break;\n default:\n if (watermark === null) {\n throw new UnrecoverableError(\n `${type} change (${msg.tag}) received before 'begin' message`,\n );\n }\n break;\n }\n\n const entry: WatermarkedChange = [watermark, change];\n unflushedBytes += this.#storer.store(entry);\n if (unflushedBytes < flushBytesThreshold) {\n // pipeline changes until flushBytesThreshold\n this.#forwarder.forward(entry);\n } else {\n // Wait for messages to clear socket buffers to ensure that they\n // make their way to subscribers. Without this `await`, the\n // messages end up being buffered in this process, which:\n // (1) results in memory pressure and increased GC activity\n // (2) prevents subscribers from processing the messages as they\n // arrive, instead getting them in a large batch after being\n // idle while they were queued (causing further delays).\n await this.#forwarder.forwardWithFlowControl(entry);\n unflushedBytes = 0;\n }\n\n if (type === 'commit' \|\| type === 'rollback') {\n watermark = null;\n }\n\n // Allow the storer to exert back pressure.\n const readyForMore = this.#storer.readyForMore();\n if (readyForMore) {\n await readyForMore;\n }\n }\n } catch (e) {\n err = e;\n } finally {\n this.#stream?.changes.cancel();\n this.#stream = undefined;\n }\n\n // When the change stream is interrupted, abort any pending transaction.\n if (watermark) {\n this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n this.#storer.abort();\n await this.#forwarder.forward([\n watermark,\n ['rollback', {tag: 'rollback'}],\n ]);\n }\n\n // Backoff and drain any pending entries in the storer before reconnecting.\n await Promise.all([\n this.#storer.stop(),\n this.#state.backoff(this.#lc, err),\n this.#state.retryDelay > 5000\n ? publishCriticalEvent(\n this.#lc,\n replicationStatusError(this.#lc, 'Replicating', err),\n )\n : promiseVoid,\n ]);\n }\n\n this.#forwarder.stopProgressMonitor();\n this.#lc.info?.('ChangeStreamer stopped');\n }\n\n async #handleControlMessage(msg: ChangeStreamControl[1]) {\n this.#lc.info?.('received control message', msg);\n const {tag} = msg;\n\n switch (tag) {\n case 'reset-required':\n await markResetRequired(this.#changeDB, this.#shard);\n await publishReplicationError(\n this.#lc,\n 'Replicating',\n msg.message ?? 'Resync required',\n msg.errorDetails,\n );\n if (this.#autoReset) {\n this.#lc.warn?.('shutting down for auto-reset');\n await this.stop(new AutoResetSignal());\n }\n break;\n default:\n unreachable(tag);\n }\n }\n\n subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n if (mode === 'serving') {\n this.#serving.resolve();\n }\n const downstream = Subscription.create<Downstream>({\n cleanup: () => this.#forwarder.remove(subscriber),\n });\n const subscriber = new Subscriber(\n protocolVersion,\n id,\n watermark,\n downstream,\n );\n if (replicaVersion !== this.#replicaVersion) {\n this.#lc.warn?.(\n `rejecting subscriber at replica version ${replicaVersion}`,\n );\n subscriber.close(\n ErrorType.WrongReplicaVersion,\n `current replica version is ${\n this.#replicaVersion\n } (requested ${replicaVersion})`,\n );\n } else {\n this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n this.#forwarder.add(subscriber);\n this.#storer.catchup(subscriber, mode);\n\n if (initial) {\n this.scheduleCleanup(watermark);\n }\n }\n return Promise.resolve(downstream);\n }\n\n scheduleCleanup(watermark: string) {\n const origSize = this.#initialWatermarks.size;\n this.#initialWatermarks.add(watermark);\n\n if (origSize === 0) {\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n\n async getChangeLogState(): Promise<{\n replicaVersion: string;\n minWatermark: string;\n }> {\n const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n if (!minWatermark) {\n this.#lc.warn?.(\n `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n );\n }\n return {\n replicaVersion: this.#replicaVersion,\n minWatermark: minWatermark ?? this.#replicaVersion,\n };\n }\n\n /\n Makes a best effort to purge the change log. In the event of a database\n * error, exceptions will be logged and swallowed, so this method is safe\n * to run in a timeout.\n /\n async #purgeOldChanges(): Promise<void> {\n const initial = [...this.#initialWatermarks];\n if (initial.length === 0) {\n this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n return;\n }\n const current = [...this.#forwarder.getAcks()];\n if (current.length === 0) {\n // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n // Bail to be safe.\n this.#lc.warn?.('No subscribers to confirm cleanup');\n return;\n }\n try {\n const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n if (earliestCurrent < earliestInitial) {\n this.#lc.info?.(\n `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n );\n } else {\n this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n const start = performance.now();\n const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n );\n this.#initialWatermarks.delete(earliestInitial);\n }\n } catch (e) {\n this.#lc.warn?.(`error purging change log`, e);\n } finally {\n if (this.#initialWatermarks.size) {\n // If there are unpurged watermarks to check, schedule the next purge.\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n }\n\n async stop(err?: unknown) {\n this.#state.stop(this.#lc, err);\n this.#stream?.changes.cancel();\n await this.#storer.stop();\n }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n// `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS 3;\n"],"names":["ErrorType.WrongReplicaVersion"],"mappings":";;;;;;;;;;;;;;;;;;AAqDA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,mBACA,WACA,iCACA,oCACA,eAAe,YACiB;AAEhC,QAAM,yBAAyB,IAAI,UAAU,KAAK;AAClD,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,EAAC,mBAAkB;AACzB,SAAO,IAAI;AAAA,IACT;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AAwJA,MAAM,mBAAoD;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,yCAAyB,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUzB,WAAW,SAAA;AAAA,EAEX,aAAa;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF;AAAA,EAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,WACA,iCACA,oCACA,eAAe,YACf;AACA,SAAK,KAAK;AACV,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AACxD,SAAK,SAAS;AACd,SAAK,YAAY;AACjB,SAAK,kBAAkB;AACvB,SAAK,UAAU;AACf,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,CAAA,aAAY,KAAK,SAAS,KAAK,KAAK,CAAC,UAAU,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;AAAA,MACxE,CAAA,QAAO,KAAK,KAAK,GAAG;AAAA,MACpB;AAAA,IAAA;AAEF,SAAK,aAAa,IAAI,UAAU,IAAI;AAAA,MAClC;AAAA,IAAA,CACD;AACD,SAAK,aAAa;AAClB,SAAK,SAAS,IAAI,aAAa,KAAK,IAAI,QAAW,YAAY;AAAA,EACjE;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,IAAI,OAAO,wBAAwB;AAExC,SAAK,WAAW,qBAAA;AAIhB,UAAM,KAAK,QAAQ,gBAAA;AAInB,UAAM,sBAAsB,wBAAwB,KAAK;AAEzD,WAAO,KAAK,OAAO,aAAa;AAC9B,UAAI;AACJ,UAAI,YAA2B;AAC/B,UAAI,iBAAiB;AACrB,UAAI;AACF,cAAM,EAAC,eAAe,iBAAA,IACpB,MAAM,KAAK,QAAQ,uCAAA;AACrB,cAAM,SAAS,MAAM,KAAK,QAAQ;AAAA,UAChC;AAAA,UACA;AAAA,QAAA;AAEF,aAAK,QAAQ,MAAM,MAAM,OAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;AAEtD,aAAK,UAAU;AACf,aAAK,OAAO,aAAA;AACZ,oBAAY;AAEZ,yBAAiB,UAAU,OAAO,SAAS;AACzC,gBAAM,CAAC,MAAM,GAAG,IAAI;AACpB,kBAAQ,MAAA;AAAA,YACN,KAAK;AACH,kBAAI,IAAI,KAAK;AACX,qBAAK,QAAQ,OAAO,MAAM;AAAA,cAC5B;AACA;AAAA,YACF,KAAK;AACH,oBAAM,KAAK,sBAAsB,GAAG;AACpC;AAAA;AAAA,YACF,KAAK;AACH,0BAAY,OAAO,CAAC,EAAE;AACtB;AAAA,YACF,KAAK;AACH,kBAAI,cAAc,OAAO,CAAC,EAAE,WAAW;AACrC,sBAAM,IAAI;AAAA,kBACR,oBAAoB,OAAO,CAAC,EAAE,SAAS,qCAAqC,SAAS;AAAA,gBAAA;AAAA,cAEzF;AACA,mBAAK,WAAW,IAAI,CAAC;AACrB;AAAA,YACF;AACE,kBAAI,cAAc,MAAM;AACtB,sBAAM,IAAI;AAAA,kBACR,GAAG,IAAI,YAAY,IAAI,GAAG;AAAA,gBAAA;AAAA,cAE9B;AACA;AAAA,UAAA;AAGJ,gBAAM,QAA2B,CAAC,WAAW,MAAM;AACnD,4BAAkB,KAAK,QAAQ,MAAM,KAAK;AAC1C,cAAI,iBAAiB,qBAAqB;AAExC,iBAAK,WAAW,QAAQ,KAAK;AAAA,UAC/B,OAAO;AAQL,kBAAM,KAAK,WAAW,uBAAuB,KAAK;AAClD,6BAAiB;AAAA,UACnB;AAEA,cAAI,SAAS,YAAY,SAAS,YAAY;AAC5C,wBAAY;AAAA,UACd;AAGA,gBAAM,eAAe,KAAK,QAAQ,aAAA;AAClC,cAAI,cAAc;AAChB,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR,UAAA;AACE,aAAK,SAAS,QAAQ,OAAA;AACtB,aAAK,UAAU;AAAA,MACjB;AAGA,UAAI,WAAW;AACb,aAAK,IAAI,OAAO,oCAAoC,SAAS,EAAE;AAC/D,aAAK,QAAQ,MAAA;AACb,cAAM,KAAK,WAAW,QAAQ;AAAA,UAC5B;AAAA,UACA,CAAC,YAAY,EAAC,KAAK,YAAW;AAAA,QAAA,CAC/B;AAAA,MACH;AAGA,YAAM,QAAQ,IAAI;AAAA,QAChB,KAAK,QAAQ,KAAA;AAAA,QACb,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AAAA,QACjC,KAAK,OAAO,aAAa,MACrB;AAAA,UACE,KAAK;AAAA,UACL,uBAAuB,KAAK,KAAK,eAAe,GAAG;AAAA,QAAA,IAErD;AAAA,MAAA,CACL;AAAA,IACH;AAEA,SAAK,WAAW,oBAAA;AAChB,SAAK,IAAI,OAAO,wBAAwB;AAAA,EAC1C;AAAA,EAEA,MAAM,sBAAsB,KAA6B;AACvD,SAAK,IAAI,OAAO,4BAA4B,GAAG;AAC/C,UAAM,EAAC,QAAO;AAEd,YAAQ,KAAA;AAAA,MACN,KAAK;AACH,cAAM,kBAAkB,KAAK,WAAW,KAAK,MAAM;AACnD,cAAM;AAAA,UACJ,KAAK;AAAA,UACL;AAAA,UACA,IAAI,WAAW;AAAA,UACf,IAAI;AAAA,QAAA;AAEN,YAAI,KAAK,YAAY;AACnB,eAAK,IAAI,OAAO,8BAA8B;AAC9C,gBAAM,KAAK,KAAK,IAAI,iBAAiB;AAAA,QACvC;AACA;AAAA,MACF;AACE,oBAAe;AAAA,IAAA;AAAA,EAErB;AAAA,EAEA,UAAU,KAAqD;AAC7D,UAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,QAAI,SAAS,WAAW;AACtB,WAAK,SAAS,QAAA;AAAA,IAChB;AACA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM,KAAK,WAAW,OAAO,UAAU;AAAA,IAAA,CACjD;AACD,UAAM,aAAa,IAAI;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,mBAAmB,KAAK,iBAAiB;AAC3C,WAAK,IAAI;AAAA,QACP,2CAA2C,cAAc;AAAA,MAAA;AAE3D,iBAAW;AAAA,QACTA;AAAAA,QACA,8BACE,KAAK,eACP,eAAe,cAAc;AAAA,MAAA;AAAA,IAEjC,OAAO;AACL,WAAK,IAAI,QAAQ,qBAAqB,WAAW,EAAE,EAAE;AAErD,WAAK,WAAW,IAAI,UAAU;AAC9B,WAAK,QAAQ,QAAQ,YAAY,IAAI;AAErC,UAAI,SAAS;AACX,aAAK,gBAAgB,SAAS;AAAA,MAChC;AAAA,IACF;AACA,WAAO,QAAQ,QAAQ,UAAU;AAAA,EACnC;AAAA,EAEA,gBAAgB,WAAmB;AACjC,UAAM,WAAW,KAAK,mBAAmB;AACzC,SAAK,mBAAmB,IAAI,SAAS;AAErC,QAAI,aAAa,GAAG;AAClB,WAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,MAAM,oBAGH;AACD,UAAM,eAAe,MAAM,KAAK,QAAQ,0BAAA;AACxC,QAAI,CAAC,cAAc;AACjB,WAAK,IAAI;AAAA,QACP;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO;AAAA,MACL,gBAAgB,KAAK;AAAA,MACrB,cAAc,gBAAgB,KAAK;AAAA,IAAA;AAAA,EAEvC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,mBAAkC;AACtC,UAAM,UAAU,CAAC,GAAG,KAAK,kBAAkB;AAC3C,QAAI,QAAQ,WAAW,GAAG;AACxB,WAAK,IAAI,OAAO,4CAA4C;AAC5D;AAAA,IACF;AACA,UAAM,UAAU,CAAC,GAAG,KAAK,WAAW,SAAS;AAC7C,QAAI,QAAQ,WAAW,GAAG;AAGxB,WAAK,IAAI,OAAO,mCAAmC;AACnD;AAAA,IACF;AACA,QAAI;AACF,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,UAAI,kBAAkB,iBAAiB;AACrC,aAAK,IAAI;AAAA,UACP,yCAAyC,eAAe,MAAM,eAAe;AAAA,QAAA;AAAA,MAEjF,OAAO;AACL,aAAK,IAAI,OAAO,0BAA0B,eAAe,MAAM;AAC/D,cAAM,QAAQ,YAAY,IAAA;AAC1B,cAAM,UAAU,MAAM,KAAK,QAAQ,mBAAmB,eAAe;AACrE,cAAM,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACrD,aAAK,IAAI;AAAA,UACP,UAAU,OAAO,mBAAmB,eAAe,KAAK,OAAO;AAAA,QAAA;AAEjE,aAAK,mBAAmB,OAAO,eAAe;AAAA,MAChD;AAAA,IACF,SAAS,GAAG;AACV,WAAK,IAAI,OAAO,4BAA4B,CAAC;AAAA,IAC/C,UAAA;AACE,UAAI,KAAK,mBAAmB,MAAM;AAEhC,aAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,KAAe;AACxB,SAAK,OAAO,KAAK,KAAK,KAAK,GAAG;AAC9B,SAAK,SAAS,QAAQ,OAAA;AACtB,UAAM,KAAK,QAAQ,KAAA;AAAA,EACrB;AACF;AAiBA,MAAM,mBAAmB,6BAA6B;"}

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts CHANGED Viewed

@@ -193,6 +193,11 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
     rowValues: v.ArrayType<v.ArrayType<v.Type<import("../../../../shared/src/bigint-json.ts").JSONValue>>>;
+    status: v.Optional<{
+        totalBytes?: number | undefined;
+        rows: number;
+        totalRows: number;
+    }>;
 }, undefined>]>, v.UnionType<[v.ObjectType<{
     tag: v.Type<"create-table">;
     spec: v.ObjectType<Omit<{
@@ -331,6 +336,11 @@ export declare const downstreamSchema: v.UnionType<[v.TupleType<[v.Type<"status"
     }, undefined>;
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
+    status: v.Optional<{
+        totalBytes?: number | undefined;
+        rows: number;
+        totalRows: number;
+    }>;
 }, undefined>]>]>]>, v.TupleType<[v.Type<"commit">, v.ObjectType<{
     tag: v.Type<"commit">;
 }, undefined>, v.ObjectType<{

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,2CAA2C,CAAC;AAEtE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;CAChE;AAqBD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;aAEvB,CAAC;AAEH,eAAO,MAAM,mBAAmB;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,qBAAsB,SAAQ,cAAc,EAAE,OAAO;IACpE;;;;OAIG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzC,iBAAiB,IAAI,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ"}
1	+ {"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AACtD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,2CAA2C,CAAC;AAEtE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;CAChE;AAqBD,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,IAAI,EAAE,cAAc,CAAC;IAErB;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY;;aAEvB,CAAC;AAEH,eAAO,MAAM,mBAAmB;;eAA+C,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,uBAAuB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,QAAA,MAAM,WAAW;;;eAAyD,CAAC;AAE3E,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAI5B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,qBAAsB,SAAQ,cAAc,EAAE,OAAO;IACpE;;;;OAIG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzC,iBAAiB,IAAI,OAAO,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ"}

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts CHANGED Viewed

@@ -1,7 +1,14 @@
+import type { LogContext } from '@rocicorp/logger';
 import type { WatermarkedChange } from './change-streamer-service.ts';
 import type { Subscriber } from './subscriber.ts';
+export type ProgressMonitorOptions = {
+    flowControlConsensusPaddingSeconds: number;
+};
 export declare class Forwarder {
     #private;
+    constructor(lc: LogContext, opts?: ProgressMonitorOptions);
+    startProgressMonitor(): void;
+    stopProgressMonitor(): void;
     /**
      * `add()` is called in lock step with `Storer.catchup()` so that the
      * two components have an equivalent interpretation of whether a Transaction is
@@ -13,8 +20,17 @@ export declare class Forwarder {
      * `forward()` is called in lockstep with `Storer.store()` so that the
      * two components have an equivalent interpretation of whether a Transaction is
      * currently being streamed.
+     *
+     * This version of forward is fire-and-forget, with no flow control. The
+     * change-streamer should call and await {@link forwardWithFlowControl()}
+     * occasionally to avoid memory blowup.
      */
-    forward(entry: WatermarkedChange): Promise<void>;
+    forward(entry: WatermarkedChange): void;
+    /**
+     * The flow-control-aware equivalent of {@link forward()}, returning a
+     * Promise that resolves when replication should continue.
+     */
+    forwardWithFlowControl(entry: WatermarkedChange): Promise<void>;
     getAcks(): Set<string>;
 }
 //# sourceMappingURL=forwarder.d.ts.map

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"~~AACA~~,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,qBAAa,SAAS;;~~IAKpB~~;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB~~;;;;OAIG~~;~~IACG~~,OAAO,CAAC,KAAK,EAAE,iBAAiB;~~IA6BtC~~,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}
1	+ {"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD,MAAM,MAAM,sBAAsB,GAAG;IACnC,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF,qBAAa,SAAS;;gBAWlB,EAAE,EAAE,UAAU,EACd,IAAI,GAAE,sBAAgE;IAMxE,oBAAoB;IAsBpB,mBAAmB;IAInB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;IAKhC;;;OAGG;IACG,sBAAsB,CAAC,KAAK,EAAE,iBAAiB;IA0CrD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC;CAQvB"}

package/out/zero-cache/src/services/change-streamer/forwarder.js CHANGED Viewed

@@ -1,8 +1,38 @@
 import { joinIterables, wrapIterable } from "../../../../shared/src/iterables.js";
+import { Broadcast } from "./broadcast.js";
 class Forwarder {
+  #lc;
+  #progressMonitorOptions;
   #active = /* @__PURE__ */ new Set();
   #queued = /* @__PURE__ */ new Set();
   #inTransaction = false;
+  #currentBroadcast;
+  #progressMonitor;
+  constructor(lc, opts = { flowControlConsensusPaddingSeconds: 1 }) {
+    this.#lc = lc.withContext("component", "progress-monitor");
+    this.#progressMonitorOptions = opts;
+  }
+  startProgressMonitor() {
+    clearInterval(this.#progressMonitor);
+    this.#progressMonitor = setInterval(this.#trackProgress, 1e3);
+  }
+  #trackProgress = () => {
+    const now = performance.now();
+    for (const sub of this.#active) {
+      sub.sampleProcessRate(now);
+    }
+    const { flowControlConsensusPaddingSeconds } = this.#progressMonitorOptions;
+    if (flowControlConsensusPaddingSeconds >= 0) {
+      this.#currentBroadcast?.checkProgress(
+        this.#lc,
+        flowControlConsensusPaddingSeconds * 1e3,
+        now
+      );
+    }
+  };
+  stopProgressMonitor() {
+    clearInterval(this.#progressMonitor);
+  }
   /**
    * `add()` is called in lock step with `Storer.catchup()` so that the
    * two components have an equivalent interpretation of whether a Transaction is
@@ -24,10 +54,29 @@ class Forwarder {
    * `forward()` is called in lockstep with `Storer.store()` so that the
    * two components have an equivalent interpretation of whether a Transaction is
    * currently being streamed.
+   *
+   * This version of forward is fire-and-forget, with no flow control. The
+   * change-streamer should call and await {@link forwardWithFlowControl()}
+   * occasionally to avoid memory blowup.
    */
-  async forward(entry) {
-    const [type] = entry[1];
-    const results = Array.from(this.#active.values(), (sub) => sub.send(entry));
+  forward(entry) {
+    Broadcast.withoutTracking(this.#active.values(), entry);
+    this.#updateActiveSubscribers(entry[1]);
+  }
+  /**
+   * The flow-control-aware equivalent of {@link forward()}, returning a
+   * Promise that resolves when replication should continue.
+   */
+  async forwardWithFlowControl(entry) {
+    const broadcast = new Broadcast(this.#active.values(), entry);
+    this.#updateActiveSubscribers(entry[1]);
+    this.#currentBroadcast = broadcast;
+    await broadcast.done;
+    if (this.#currentBroadcast === broadcast) {
+      this.#currentBroadcast = void 0;
+    }
+  }
+  #updateActiveSubscribers([type]) {
     switch (type) {
       case "begin":
         this.#inTransaction = true;
@@ -41,7 +90,6 @@ class Forwarder {
         this.#queued.clear();
         break;
     }
-    await Promise.all(results);
   }
   getAcks() {
     return new Set(

package/out/zero-cache/src/services/change-streamer/forwarder.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"forwarder.js","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport class Forwarder {\n readonly #active = new Set<Subscriber>();\n readonly #queued = new Set<Subscriber>();\n #inTransaction = false;\n\n /*\n `add()` is called in lock step with `Storer.catchup()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n /\n add(sub: Subscriber) {\n if (this.#inTransaction) {\n this.#queued.add(sub);\n } else {\n this.#active.add(sub);\n }\n }\n\n remove(sub: Subscriber) {\n this.#active.delete(sub);\n this.#queued.delete(sub);\n sub.close();\n }\n\n /\n `forward()` is called in lockstep with `Storer.store()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n */\n ~~async~~ forward(entry: WatermarkedChange) {\n ~~const~~ ~~[type] =~~ entry[1];\n const ~~results~~ = ~~Array.from~~(this.#active.values(), ~~sub~~ => ~~sub~~.~~send~~(~~entry~~));\n switch (type) {\n case 'begin':\n // While in a Transaction, all added subscribers are \"queued\" so that no\n // messages are forwarded to them. This state corresponds to being queued\n // for catchup in the Storer, which will retrieve historic changes\n // and call catchup() once the current transaction is committed.\n this.#inTransaction = true;\n break;\n case 'commit':\n case 'rollback':\n // Upon commit or rollback, all queued subscribers are transferred to\n // the active set. This means that they can receive messages starting\n // from the next transaction.\n //\n // Note that if catchup is still in progress (in the Storer), these messages\n // will be buffered in the backlog until catchup completes.\n this.#inTransaction = false;\n for (const sub of this.#queued.values()) {\n this.#active.add(sub);\n }\n this.#queued.clear();\n break;\n }\n ~~await Promise.all(results);\n~~ }\n\n getAcks(): Set<string> {\n return new Set(\n joinIterables(\n wrapIterable(this.#active).map(s => s.acked),\n wrapIterable(this.#queued).map(s => s.acked),\n ),\n );\n }\n}\n"],"names":[],"mappings":"~~;AAIO~~,MAAM,UAAU;AAAA,EACZ,8BAAc,IAAA;AAAA,EACd,8BAAc,IAAA;AAAA,EACvB,iBAAiB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,~~EAOjB~~,IAAI,KAAiB;AACnB,QAAI,KAAK,gBAAgB;AACvB,WAAK,QAAQ,IAAI,GAAG;AAAA,IACtB,OAAO;AACL,WAAK,QAAQ,IAAI,GAAG;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,OAAO,KAAiB;AACtB,SAAK,QAAQ,OAAO,GAAG;AACvB,SAAK,QAAQ,OAAO,GAAG;AACvB,QAAI,MAAA;AAAA,EACN;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,~~EAOA~~,~~MAAM,~~QAAQ,OAA0B;~~AACtC~~,~~UAAM~~,~~CAAC~~,~~IAAI~~,~~IAAI~~,MAAM,CAAC;~~AACtB~~,~~UAAM~~,~~UAAU~~,MAAM,~~KAAK~~,KAAK,QAAQ,OAAA,GAAU,~~CAAA~~,~~QAAO~~,~~IAAI~~,~~KAAK~~,KAAK,CAAC;~~AACxE~~,YAAQ,MAAA;AAAA,MACN,KAAK;AAKH,aAAK,iBAAiB;AACtB;AAAA,MACF,KAAK;AAAA,MACL,KAAK;AAOH,aAAK,iBAAiB;AACtB,mBAAW,OAAO,KAAK,QAAQ,OAAA,GAAU;AACvC,eAAK,QAAQ,IAAI,GAAG;AAAA,QACtB;AACA,aAAK,QAAQ,MAAA;AACb;AAAA,IAAA;~~AAEJ,UAAM,QAAQ,IAAI,OAAO;~~AAAA,~~EAC3B~~;AAAA,EAEA,UAAuB;AACrB,WAAO,IAAI;AAAA,MACT;AAAA,QACE,aAAa,KAAK,OAAO,EAAE,IAAI,CAAA,MAAK,EAAE,KAAK;AAAA,QAC3C,aAAa,KAAK,OAAO,EAAE,IAAI,CAAA,MAAK,EAAE,KAAK;AAAA,MAAA;AAAA,IAC7C;AAAA,EAEJ;AACF;"}
1	+ {"version":3,"file":"forwarder.js","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {joinIterables, wrapIterable} from '../../../../shared/src/iterables.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport {Broadcast} from './broadcast.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport type ProgressMonitorOptions = {\n flowControlConsensusPaddingSeconds: number;\n};\n\nexport class Forwarder {\n readonly #lc: LogContext;\n readonly #progressMonitorOptions: ProgressMonitorOptions;\n readonly #active = new Set<Subscriber>();\n readonly #queued = new Set<Subscriber>();\n #inTransaction = false;\n\n #currentBroadcast: Broadcast \| undefined;\n #progressMonitor: NodeJS.Timeout \| undefined;\n\n constructor(\n lc: LogContext,\n opts: ProgressMonitorOptions = {flowControlConsensusPaddingSeconds: 1},\n ) {\n this.#lc = lc.withContext('component', 'progress-monitor');\n this.#progressMonitorOptions = opts;\n }\n\n startProgressMonitor() {\n clearInterval(this.#progressMonitor);\n this.#progressMonitor = setInterval(this.#trackProgress, 1000);\n }\n\n readonly #trackProgress = () => {\n const now = performance.now();\n for (const sub of this.#active) {\n sub.sampleProcessRate(now);\n }\n\n const {flowControlConsensusPaddingSeconds} = this.#progressMonitorOptions;\n // A negative number disables early flow control release.\n if (flowControlConsensusPaddingSeconds >= 0) {\n this.#currentBroadcast?.checkProgress(\n this.#lc,\n flowControlConsensusPaddingSeconds * 1000,\n now,\n );\n }\n };\n\n stopProgressMonitor() {\n clearInterval(this.#progressMonitor);\n }\n\n /*\n `add()` is called in lock step with `Storer.catchup()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n /\n add(sub: Subscriber) {\n if (this.#inTransaction) {\n this.#queued.add(sub);\n } else {\n this.#active.add(sub);\n }\n }\n\n remove(sub: Subscriber) {\n this.#active.delete(sub);\n this.#queued.delete(sub);\n sub.close();\n }\n\n /\n `forward()` is called in lockstep with `Storer.store()` so that the\n * two components have an equivalent interpretation of whether a Transaction is\n * currently being streamed.\n \n This version of forward is fire-and-forget, with no flow control. The\n * change-streamer should call and await {@link forwardWithFlowControl()}\n * occasionally to avoid memory blowup.\n /\n forward(entry: WatermarkedChange) {\n Broadcast.withoutTracking(this.#active.values(), entry);\n this.#updateActiveSubscribers(entry[1]);\n }\n\n /\n The flow-control-aware equivalent of {@link forward()}, returning a\n * Promise that resolves when replication should continue.\n */\n async forwardWithFlowControl(entry: WatermarkedChange) {\n const broadcast = new Broadcast(this.#active.values(), entry);\n this.#updateActiveSubscribers(entry[1]);\n\n // set for progress tracking\n this.#currentBroadcast = broadcast;\n\n await broadcast.done;\n\n // Technically #currentBroadcast may have changed, so only\n // unset if it if is still the same.\n if (this.#currentBroadcast === broadcast) {\n this.#currentBroadcast = undefined;\n }\n }\n\n #updateActiveSubscribers([type]: ChangeStreamData) {\n switch (type) {\n case 'begin':\n // While in a Transaction, all added subscribers are \"queued\" so that no\n // messages are forwarded to them. This state corresponds to being queued\n // for catchup in the Storer, which will retrieve historic changes\n // and call catchup() once the current transaction is committed.\n this.#inTransaction = true;\n break;\n case 'commit':\n case 'rollback':\n // Upon commit or rollback, all queued subscribers are transferred to\n // the active set. This means that they can receive messages starting\n // from the next transaction.\n //\n // Note that if catchup is still in progress (in the Storer), these messages\n // will be buffered in the backlog until catchup completes.\n this.#inTransaction = false;\n for (const sub of this.#queued.values()) {\n this.#active.add(sub);\n }\n this.#queued.clear();\n break;\n }\n }\n\n getAcks(): Set<string> {\n return new Set(\n joinIterables(\n wrapIterable(this.#active).map(s => s.acked),\n wrapIterable(this.#queued).map(s => s.acked),\n ),\n );\n }\n}\n"],"names":[],"mappings":";;AAWO,MAAM,UAAU;AAAA,EACZ;AAAA,EACA;AAAA,EACA,8BAAc,IAAA;AAAA,EACd,8BAAc,IAAA;AAAA,EACvB,iBAAiB;AAAA,EAEjB;AAAA,EACA;AAAA,EAEA,YACE,IACA,OAA+B,EAAC,oCAAoC,KACpE;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,kBAAkB;AACzD,SAAK,0BAA0B;AAAA,EACjC;AAAA,EAEA,uBAAuB;AACrB,kBAAc,KAAK,gBAAgB;AACnC,SAAK,mBAAmB,YAAY,KAAK,gBAAgB,GAAI;AAAA,EAC/D;AAAA,EAES,iBAAiB,MAAM;AAC9B,UAAM,MAAM,YAAY,IAAA;AACxB,eAAW,OAAO,KAAK,SAAS;AAC9B,UAAI,kBAAkB,GAAG;AAAA,IAC3B;AAEA,UAAM,EAAC,uCAAsC,KAAK;AAElD,QAAI,sCAAsC,GAAG;AAC3C,WAAK,mBAAmB;AAAA,QACtB,KAAK;AAAA,QACL,qCAAqC;AAAA,QACrC;AAAA,MAAA;AAAA,IAEJ;AAAA,EACF;AAAA,EAEA,sBAAsB;AACpB,kBAAc,KAAK,gBAAgB;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,IAAI,KAAiB;AACnB,QAAI,KAAK,gBAAgB;AACvB,WAAK,QAAQ,IAAI,GAAG;AAAA,IACtB,OAAO;AACL,WAAK,QAAQ,IAAI,GAAG;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,OAAO,KAAiB;AACtB,SAAK,QAAQ,OAAO,GAAG;AACvB,SAAK,QAAQ,OAAO,GAAG;AACvB,QAAI,MAAA;AAAA,EACN;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,QAAQ,OAA0B;AAChC,cAAU,gBAAgB,KAAK,QAAQ,OAAA,GAAU,KAAK;AACtD,SAAK,yBAAyB,MAAM,CAAC,CAAC;AAAA,EACxC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,uBAAuB,OAA0B;AACrD,UAAM,YAAY,IAAI,UAAU,KAAK,QAAQ,OAAA,GAAU,KAAK;AAC5D,SAAK,yBAAyB,MAAM,CAAC,CAAC;AAGtC,SAAK,oBAAoB;AAEzB,UAAM,UAAU;AAIhB,QAAI,KAAK,sBAAsB,WAAW;AACxC,WAAK,oBAAoB;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,yBAAyB,CAAC,IAAI,GAAqB;AACjD,YAAQ,MAAA;AAAA,MACN,KAAK;AAKH,aAAK,iBAAiB;AACtB;AAAA,MACF,KAAK;AAAA,MACL,KAAK;AAOH,aAAK,iBAAiB;AACtB,mBAAW,OAAO,KAAK,QAAQ,OAAA,GAAU;AACvC,eAAK,QAAQ,IAAI,GAAG;AAAA,QACtB;AACA,aAAK,QAAQ,MAAA;AACb;AAAA,IAAA;AAAA,EAEN;AAAA,EAEA,UAAuB;AACrB,WAAO,IAAI;AAAA,MACT;AAAA,QACE,aAAa,KAAK,OAAO,EAAE,IAAI,CAAA,MAAK,EAAE,KAAK;AAAA,QAC3C,aAAa,KAAK,OAAO,EAAE,IAAI,CAAA,MAAK,EAAE,KAAK;AAAA,MAAA;AAAA,IAC7C;AAAA,EAEJ;AACF;"}

package/out/zero-cache/src/services/change-streamer/subscriber.d.ts CHANGED Viewed

@@ -26,6 +26,24 @@ export declare class Subscriber {
      * entries that were received during the catchup.
      */
     setCaughtUp(): void;
+    /**
+     * The number of downstream messages that have yet to be acked.
+     */
+    get numPending(): number;
+    /**
+     * The total number of downstream messages that the subscriber has
+     * processed (i.e. acked).
+     */
+    get numProcessed(): number;
+    /**
+     * Records a new history entry for the number of messages processed,
+     * keeping the number of samples bounded to `maxSamples`.
+     */
+    sampleProcessRate(now: number, maxSamples?: number): this;
+    getStats(): {
+        processRate: number;
+        pending: number;
+    };
     supportsMessage(change: ChangeStreamData[1]): boolean;
     fail(err?: unknown): void;
     close(error?: ErrorType, message?: string): void;

package/out/zero-cache/src/services/change-streamer/subscriber.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"subscriber.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;~~AAEzD~~,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,sCAAsC,CAAC;AAC3E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAElD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC;;;;;;GAMG;AACH,qBAAa,UAAU;;IAErB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAOlB,eAAe,EAAE,MAAM,EACvB,EAAE,EAAE,MAAM,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,YAAY,CAAC,UAAU,CAAC;IAUtC,IAAI,SAAS,WAEZ;IAED,IAAI,KAAK,WAER;IAEK,IAAI,CAAC,MAAM,EAAE,iBAAiB;IAoBpC,kEAAkE;IAC5D,OAAO,CAAC,MAAM,EAAE,iBAAiB;IAKvC;;;OAGG;IACH,WAAW;~~IAoCX~~,eAAe,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAS3C,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO;IAIlB,KAAK,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM;CAS1C"}
1	+ {"version":3,"file":"subscriber.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,gCAAgC,CAAC;AAGzD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,sCAAsC,CAAC;AAC3E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAC;AAElD,KAAK,SAAS,GAAG,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;AAExC;;;;;;GAMG;AACH,qBAAa,UAAU;;IAErB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAOlB,eAAe,EAAE,MAAM,EACvB,EAAE,EAAE,MAAM,EACV,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,YAAY,CAAC,UAAU,CAAC;IAUtC,IAAI,SAAS,WAEZ;IAED,IAAI,KAAK,WAER;IAEK,IAAI,CAAC,MAAM,EAAE,iBAAiB;IAoBpC,kEAAkE;IAC5D,OAAO,CAAC,MAAM,EAAE,iBAAiB;IAKvC;;;OAGG;IACH,WAAW;IA0DX;;OAEG;IACH,IAAI,UAAU,WAEb;IAED;;;OAGG;IACH,IAAI,YAAY,WAEf;IAED;;;OAGG;IACH,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,SAAK,GAAG,IAAI;IAQrD,QAAQ,IAAI;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAC;IAalD,eAAe,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAS3C,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO;IAIlB,KAAK,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,MAAM;CAS1C"}

package/out/zero-cache/src/services/change-streamer/subscriber.js CHANGED Viewed

@@ -1,4 +1,5 @@
 import { assert } from "../../../../shared/src/asserts.js";
+import { must } from "../../../../shared/src/must.js";
 import { max } from "../../types/lexi-version.js";
 import "./change-streamer.js";
 import { Unknown } from "./error-type-enum.js";
@@ -29,21 +30,21 @@ class Subscriber {
       if (this.#backlog) {
         this.#backlog.push(change);
       } else {
-        await this.#send(change);
+        await this.#sendChange(change);
       }
     }
   }
   #initialStatusSent = false;
   #ensureInitialStatusSent() {
     if (this.#protocolVersion >= 2 && !this.#initialStatusSent) {
-      this.#downstream.push(["status", { tag: "status" }]);
+      void this.#sendDownstream(["status", { tag: "status" }]);
       this.#initialStatusSent = true;
     }
   }
   /** catchup() is called on ChangeEntries loaded from the store. */
   async catchup(change) {
     this.#ensureInitialStatusSent();
-    await this.#send(change);
+    await this.#sendChange(change);
   }
   /**
    * Marks the Subscribe as "caught up" and flushes any backlog of
@@ -56,11 +57,11 @@ class Subscriber {
       "setCaughtUp() called but subscriber is not in catchup mode"
     );
     for (const change of this.#backlog) {
-      void this.#send(change);
+      void this.#sendChange(change);
     }
     this.#backlog = null;
   }
-  async #send(change) {
+  async #sendChange(change) {
     const [watermark, downstream] = change;
     if (watermark <= this.watermark) {
       return;
@@ -68,16 +69,71 @@ class Subscriber {
     if (!this.supportsMessage(downstream[1])) {
       return;
     }
-    const pending = this.#downstream.push(downstream);
     if (downstream[0] === "commit") {
       this.#watermark = watermark;
-      void pending.result.then((val) => {
-        if (val === "consumed") {
-          this.#acked = max(this.#acked, watermark);
-        }
-      });
     }
-    await pending.result;
+    const result = await this.#sendDownstream(downstream);
+    if (downstream[0] === "commit" && result === "consumed") {
+      this.#acked = max(this.#acked, watermark);
+    }
+  }
+  async #sendDownstream(downstream) {
+    this.#pending++;
+    const { result } = this.#downstream.push(downstream);
+    try {
+      return await result;
+    } finally {
+      this.#pending--;
+      this.#processed++;
+    }
+  }
+  // `pending` and `processed` stats are tracked by periodically sampling
+  // the running totals (by the progress tracker in the Forwarder).
+  // This information was originally collected for use in flow control
+  // decisions. The final flow control algorithm ended up being simpler
+  // than expected and does not actually use this information. However, the
+  // stats are still tracked and logged during flow control decisions for
+  // debugging, forensics, and potential improvements to the algorithm.
+  #pending = 0;
+  #processed = 0;
+  #samples = [
+    { processed: 0, timestamp: performance.now() }
+  ];
+  /**
+   * The number of downstream messages that have yet to be acked.
+   */
+  get numPending() {
+    return this.#pending;
+  }
+  /**
+   * The total number of downstream messages that the subscriber has
+   * processed (i.e. acked).
+   */
+  get numProcessed() {
+    return this.#processed;
+  }
+  /**
+   * Records a new history entry for the number of messages processed,
+   * keeping the number of samples bounded to `maxSamples`.
+   */
+  sampleProcessRate(now, maxSamples = 10) {
+    while (this.#samples.length >= maxSamples) {
+      this.#samples.shift();
+    }
+    this.#samples.push({ processed: this.#processed, timestamp: now });
+    return this;
+  }
+  getStats() {
+    const pending = this.#pending;
+    if (this.#samples.length < 2) {
+      return { processRate: 0, pending };
+    }
+    const from = this.#samples[0];
+    const to = must(this.#samples.at(-1));
+    const processed = to.processed - from.processed;
+    const seconds = (to.timestamp - from.timestamp) / 1e3;
+    const processRate = seconds === 0 ? 0 : processed / seconds;
+    return { processRate, pending };
   }
   supportsMessage(change) {
     switch (change.tag) {

package/out/zero-cache/src/services/change-streamer/subscriber.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"subscriber.js","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type Downstream} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/*\n Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n /\nexport class Subscriber {\n readonly #protocolVersion: number;\n readonly id: string;\n readonly #downstream: Subscription<Downstream>;\n #watermark: string;\n #acked: string;\n #backlog: WatermarkedChange[] \| null;\n\n constructor(\n protocolVersion: number,\n id: string,\n watermark: string,\n downstream: Subscription<Downstream>,\n ) {\n this.#protocolVersion = protocolVersion;\n this.id = id;\n this.#downstream = downstream;\n this.#watermark = watermark;\n this.#acked = watermark;\n this.#backlog = [];\n }\n\n get watermark() {\n return this.#watermark;\n }\n\n get acked() {\n return this.#acked;\n }\n\n async send(change: WatermarkedChange) {\n const [watermark] = change;\n if (watermark > this.#watermark) {\n if (this.#backlog) {\n this.#backlog.push(change);\n } else {\n await this.#~~send~~(change);\n }\n }\n }\n\n #initialStatusSent = false;\n\n #ensureInitialStatusSent() {\n if (this.#protocolVersion >= 2 && !this.#initialStatusSent) {\n this.#~~downstream.push~~(['status', {tag: 'status'}]);\n this.#initialStatusSent = true;\n }\n }\n\n /* catchup() is called on ChangeEntries loaded from the store. /\n async catchup(change: WatermarkedChange) {\n this.#ensureInitialStatusSent();\n await this.#~~send~~(change);\n }\n\n /\n Marks the Subscribe as \"caught up\" and flushes any backlog of\n * entries that were received during the catchup.\n */\n setCaughtUp() {\n this.#ensureInitialStatusSent();\n assert(\n this.#backlog,\n 'setCaughtUp() called but subscriber is not in catchup mode',\n );\n // Note that this method must be asynchronous in order for send() to\n // interpret the #backlog variable correctly. This is the only place\n // where I/O flow control is not heeded. However, it will be awaited\n // by the next caller to send().\n for (const change of this.#backlog) {\n void this.#~~send~~(change);\n }\n this.#backlog = null;\n }\n\n async #~~send~~(change: WatermarkedChange) {\n const [watermark, downstream] = change;\n if (watermark <= this.watermark) {\n return;\n }\n if (!this.supportsMessage(downstream[1])) {\n return;\n }\n const ~~pending~~ = this.#~~downstream.push~~(downstream);\n if (downstream[0] === 'commit') {\n this.#~~watermark~~ = watermark;\n ~~void~~ pending.result.~~then~~(~~val~~ => {\n if (~~val~~ ~~===~~ ~~'consumed'~~) {\n this.#acked = ~~max~~(this.#~~acked,~~ ~~watermark~~);\n }\n });\n }\n ~~await~~ pending.~~result~~;\n }\n\n supportsMessage(change: ChangeStreamData[1]) {\n switch (change.tag) {\n case 'update-table-metadata':\n // update-table-row-key is only understood by subscribers >= protocol v5\n return this.#protocolVersion >= 5;\n }\n return true;\n }\n\n fail(err?: unknown) {\n this.close(ErrorType.Unknown, String(err));\n }\n\n close(error?: ErrorType, message?: string) {\n if (error) {\n const {result} = this.#downstream.push(['error', {type: error, message}]);\n // Wait for the ACK of the error message before closing the connection.\n void result.then(() => this.#downstream.cancel());\n } else {\n this.#downstream.cancel();\n }\n }\n}\n"],"names":["ErrorType.Unknown"],"mappings":"~~;;;;AAkBO~~,MAAM,WAAW;AAAA,EACb;AAAA,EACA;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,iBACA,IACA,WACA,YACA;AACA,SAAK,mBAAmB;AACxB,SAAK,KAAK;AACV,SAAK,cAAc;AACnB,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,WAAW,CAAA;AAAA,EAClB;AAAA,EAEA,IAAI,YAAY;AACd,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAAQ;AACV,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,KAAK,QAA2B;AACpC,UAAM,CAAC,SAAS,IAAI;AACpB,QAAI,YAAY,KAAK,YAAY;AAC/B,UAAI,KAAK,UAAU;AACjB,aAAK,SAAS,KAAK,MAAM;AAAA,MAC3B,OAAO;AACL,cAAM,KAAK,~~MAAM~~,MAAM;AAAA,~~MACzB~~;AAAA,IACF;AAAA,EACF;AAAA,EAEA,qBAAqB;AAAA,EAErB,2BAA2B;AACzB,QAAI,KAAK,oBAAoB,KAAK,CAAC,KAAK,oBAAoB;AAC1D,WAAK,~~YAAY,~~KAAK,CAAC,UAAU,EAAC,KAAK,SAAA,CAAS,CAAC;~~AACjD~~,WAAK,qBAAqB;AAAA,IAC5B;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,QAAQ,QAA2B;AACvC,SAAK,yBAAA;AACL,UAAM,KAAK,~~MAAM~~,MAAM;AAAA,~~EACzB~~;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,cAAc;AACZ,SAAK,yBAAA;AACL;AAAA,MACE,KAAK;AAAA,MACL;AAAA,IAAA;AAMF,eAAW,UAAU,KAAK,UAAU;AAClC,WAAK,KAAK,~~MAAM~~,MAAM;AAAA,~~IACxB~~;AACA,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,~~MAAM~~,QAA2B;~~AACrC~~,UAAM,CAAC,WAAW,UAAU,IAAI;AAChC,QAAI,aAAa,KAAK,WAAW;AAC/B;AAAA,IACF;AACA,QAAI,CAAC,KAAK,gBAAgB,WAAW,CAAC,CAAC,GAAG;AACxC;AAAA,IACF;AACA,~~UAAM~~,UAAU,~~KAAK~~,~~YAAY~~,KAAK,UAAU;~~AAChD~~,QAAI,WAAW,CAAC,MAAM,UAAU;~~AAC9B~~,WAAK,aAAa;~~AAClB~~,~~WAAK~~,~~QAAQ~~,~~OAAO~~,KAAK,~~CAAA~~,~~QAAO~~;~~AAC9B~~,~~YAAI~~,~~QAAQ~~,YAAY;~~AACtB~~,~~eAAK~~,SAAS,~~IAAI~~,KAAK,~~QAAQ~~,~~SAAS~~;AAAA,~~QAC1C~~;AAAA,~~MACF~~,~~CAAC~~;AAAA,~~IACH~~;AACA,UAAM,~~QAAQ~~;AAAA,~~EAChB~~;AAAA,EAEA,gBAAgB,QAA6B;AAC3C,YAAQ,OAAO,KAAA;AAAA,MACb,KAAK;AAEH,eAAO,KAAK,oBAAoB;AAAA,IAAA;AAEpC,WAAO;AAAA,EACT;AAAA,EAEA,KAAK,KAAe;AAClB,SAAK,MAAMA,SAAmB,OAAO,GAAG,CAAC;AAAA,EAC3C;AAAA,EAEA,MAAM,OAAmB,SAAkB;AACzC,QAAI,OAAO;AACT,YAAM,EAAC,OAAA,IAAU,KAAK,YAAY,KAAK,CAAC,SAAS,EAAC,MAAM,OAAO,QAAA,CAAQ,CAAC;AAExE,WAAK,OAAO,KAAK,MAAM,KAAK,YAAY,QAAQ;AAAA,IAClD,OAAO;AACL,WAAK,YAAY,OAAA;AAAA,IACnB;AAAA,EACF;AACF;"}
1	+ {"version":3,"file":"subscriber.js","sources":["../../../../../../zero-cache/src/services/change-streamer/subscriber.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {max} from '../../types/lexi-version.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type Downstream} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/*\n Encapsulates a subscriber to changes. All subscribers start in a\n * \"catchup\" phase in which changes are buffered in a backlog while the\n * storer is queried to send any changes that were committed since the\n * subscriber's watermark. Once the catchup is complete, calls to\n * {@link send()} result in immediately sending the change.\n /\nexport class Subscriber {\n readonly #protocolVersion: number;\n readonly id: string;\n readonly #downstream: Subscription<Downstream>;\n #watermark: string;\n #acked: string;\n #backlog: WatermarkedChange[] \| null;\n\n constructor(\n protocolVersion: number,\n id: string,\n watermark: string,\n downstream: Subscription<Downstream>,\n ) {\n this.#protocolVersion = protocolVersion;\n this.id = id;\n this.#downstream = downstream;\n this.#watermark = watermark;\n this.#acked = watermark;\n this.#backlog = [];\n }\n\n get watermark() {\n return this.#watermark;\n }\n\n get acked() {\n return this.#acked;\n }\n\n async send(change: WatermarkedChange) {\n const [watermark] = change;\n if (watermark > this.#watermark) {\n if (this.#backlog) {\n this.#backlog.push(change);\n } else {\n await this.#sendChange(change);\n }\n }\n }\n\n #initialStatusSent = false;\n\n #ensureInitialStatusSent() {\n if (this.#protocolVersion >= 2 && !this.#initialStatusSent) {\n void this.#sendDownstream(['status', {tag: 'status'}]);\n this.#initialStatusSent = true;\n }\n }\n\n /* catchup() is called on ChangeEntries loaded from the store. /\n async catchup(change: WatermarkedChange) {\n this.#ensureInitialStatusSent();\n await this.#sendChange(change);\n }\n\n /\n Marks the Subscribe as \"caught up\" and flushes any backlog of\n * entries that were received during the catchup.\n /\n setCaughtUp() {\n this.#ensureInitialStatusSent();\n assert(\n this.#backlog,\n 'setCaughtUp() called but subscriber is not in catchup mode',\n );\n // Note that this method must be asynchronous in order for send() to\n // interpret the #backlog variable correctly. This is the only place\n // where I/O flow control is not heeded. However, it will be awaited\n // by the next caller to send().\n for (const change of this.#backlog) {\n void this.#sendChange(change);\n }\n this.#backlog = null;\n }\n\n async #sendChange(change: WatermarkedChange) {\n const [watermark, downstream] = change;\n if (watermark <= this.watermark) {\n return;\n }\n if (!this.supportsMessage(downstream[1])) {\n return;\n }\n if (downstream[0] === 'commit') {\n this.#watermark = watermark;\n }\n const result = await this.#sendDownstream(downstream);\n if (downstream[0] === 'commit' && result === 'consumed') {\n this.#acked = max(this.#acked, watermark);\n }\n }\n\n async #sendDownstream(downstream: Downstream) {\n this.#pending++;\n const {result} = this.#downstream.push(downstream);\n try {\n return await result;\n } finally {\n this.#pending--;\n this.#processed++;\n }\n }\n\n // `pending` and `processed` stats are tracked by periodically sampling\n // the running totals (by the progress tracker in the Forwarder).\n // This information was originally collected for use in flow control\n // decisions. The final flow control algorithm ended up being simpler\n // than expected and does not actually use this information. However, the\n // stats are still tracked and logged during flow control decisions for\n // debugging, forensics, and potential improvements to the algorithm.\n\n #pending = 0;\n #processed = 0;\n #samples: {processed: number; timestamp: number}[] = [\n {processed: 0, timestamp: performance.now()},\n ];\n\n /\n The number of downstream messages that have yet to be acked.\n /\n get numPending() {\n return this.#pending;\n }\n\n /\n The total number of downstream messages that the subscriber has\n * processed (i.e. acked).\n /\n get numProcessed() {\n return this.#processed;\n }\n\n /\n Records a new history entry for the number of messages processed,\n * keeping the number of samples bounded to `maxSamples`.\n */\n sampleProcessRate(now: number, maxSamples = 10): this {\n while (this.#samples.length >= maxSamples) {\n this.#samples.shift();\n }\n this.#samples.push({processed: this.#processed, timestamp: now});\n return this;\n }\n\n getStats(): {processRate: number; pending: number} {\n const pending = this.#pending;\n if (this.#samples.length < 2) {\n return {processRate: 0, pending};\n }\n const from = this.#samples[0];\n const to = must(this.#samples.at(-1));\n const processed = to.processed - from.processed;\n const seconds = (to.timestamp - from.timestamp) / 1000;\n const processRate = seconds === 0 ? 0 : processed / seconds;\n return {processRate, pending};\n }\n\n supportsMessage(change: ChangeStreamData[1]) {\n switch (change.tag) {\n case 'update-table-metadata':\n // update-table-row-key is only understood by subscribers >= protocol v5\n return this.#protocolVersion >= 5;\n }\n return true;\n }\n\n fail(err?: unknown) {\n this.close(ErrorType.Unknown, String(err));\n }\n\n close(error?: ErrorType, message?: string) {\n if (error) {\n const {result} = this.#downstream.push(['error', {type: error, message}]);\n // Wait for the ACK of the error message before closing the connection.\n void result.then(() => this.#downstream.cancel());\n } else {\n this.#downstream.cancel();\n }\n }\n}\n"],"names":["ErrorType.Unknown"],"mappings":";;;;;AAmBO,MAAM,WAAW;AAAA,EACb;AAAA,EACA;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,iBACA,IACA,WACA,YACA;AACA,SAAK,mBAAmB;AACxB,SAAK,KAAK;AACV,SAAK,cAAc;AACnB,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,WAAW,CAAA;AAAA,EAClB;AAAA,EAEA,IAAI,YAAY;AACd,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAAQ;AACV,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,KAAK,QAA2B;AACpC,UAAM,CAAC,SAAS,IAAI;AACpB,QAAI,YAAY,KAAK,YAAY;AAC/B,UAAI,KAAK,UAAU;AACjB,aAAK,SAAS,KAAK,MAAM;AAAA,MAC3B,OAAO;AACL,cAAM,KAAK,YAAY,MAAM;AAAA,MAC/B;AAAA,IACF;AAAA,EACF;AAAA,EAEA,qBAAqB;AAAA,EAErB,2BAA2B;AACzB,QAAI,KAAK,oBAAoB,KAAK,CAAC,KAAK,oBAAoB;AAC1D,WAAK,KAAK,gBAAgB,CAAC,UAAU,EAAC,KAAK,SAAA,CAAS,CAAC;AACrD,WAAK,qBAAqB;AAAA,IAC5B;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,QAAQ,QAA2B;AACvC,SAAK,yBAAA;AACL,UAAM,KAAK,YAAY,MAAM;AAAA,EAC/B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,cAAc;AACZ,SAAK,yBAAA;AACL;AAAA,MACE,KAAK;AAAA,MACL;AAAA,IAAA;AAMF,eAAW,UAAU,KAAK,UAAU;AAClC,WAAK,KAAK,YAAY,MAAM;AAAA,IAC9B;AACA,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,YAAY,QAA2B;AAC3C,UAAM,CAAC,WAAW,UAAU,IAAI;AAChC,QAAI,aAAa,KAAK,WAAW;AAC/B;AAAA,IACF;AACA,QAAI,CAAC,KAAK,gBAAgB,WAAW,CAAC,CAAC,GAAG;AACxC;AAAA,IACF;AACA,QAAI,WAAW,CAAC,MAAM,UAAU;AAC9B,WAAK,aAAa;AAAA,IACpB;AACA,UAAM,SAAS,MAAM,KAAK,gBAAgB,UAAU;AACpD,QAAI,WAAW,CAAC,MAAM,YAAY,WAAW,YAAY;AACvD,WAAK,SAAS,IAAI,KAAK,QAAQ,SAAS;AAAA,IAC1C;AAAA,EACF;AAAA,EAEA,MAAM,gBAAgB,YAAwB;AAC5C,SAAK;AACL,UAAM,EAAC,OAAA,IAAU,KAAK,YAAY,KAAK,UAAU;AACjD,QAAI;AACF,aAAO,MAAM;AAAA,IACf,UAAA;AACE,WAAK;AACL,WAAK;AAAA,IACP;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,WAAW;AAAA,EACX,aAAa;AAAA,EACb,WAAqD;AAAA,IACnD,EAAC,WAAW,GAAG,WAAW,YAAY,MAAI;AAAA,EAAC;AAAA;AAAA;AAAA;AAAA,EAM7C,IAAI,aAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,IAAI,eAAe;AACjB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,kBAAkB,KAAa,aAAa,IAAU;AACpD,WAAO,KAAK,SAAS,UAAU,YAAY;AACzC,WAAK,SAAS,MAAA;AAAA,IAChB;AACA,SAAK,SAAS,KAAK,EAAC,WAAW,KAAK,YAAY,WAAW,KAAI;AAC/D,WAAO;AAAA,EACT;AAAA,EAEA,WAAmD;AACjD,UAAM,UAAU,KAAK;AACrB,QAAI,KAAK,SAAS,SAAS,GAAG;AAC5B,aAAO,EAAC,aAAa,GAAG,QAAA;AAAA,IAC1B;AACA,UAAM,OAAO,KAAK,SAAS,CAAC;AAC5B,UAAM,KAAK,KAAK,KAAK,SAAS,GAAG,EAAE,CAAC;AACpC,UAAM,YAAY,GAAG,YAAY,KAAK;AACtC,UAAM,WAAW,GAAG,YAAY,KAAK,aAAa;AAClD,UAAM,cAAc,YAAY,IAAI,IAAI,YAAY;AACpD,WAAO,EAAC,aAAa,QAAA;AAAA,EACvB;AAAA,EAEA,gBAAgB,QAA6B;AAC3C,YAAQ,OAAO,KAAA;AAAA,MACb,KAAK;AAEH,eAAO,KAAK,oBAAoB;AAAA,IAAA;AAEpC,WAAO;AAAA,EACT;AAAA,EAEA,KAAK,KAAe;AAClB,SAAK,MAAMA,SAAmB,OAAO,GAAG,CAAC;AAAA,EAC3C;AAAA,EAEA,MAAM,OAAmB,SAAkB;AACzC,QAAI,OAAO;AACT,YAAM,EAAC,OAAA,IAAU,KAAK,YAAY,KAAK,CAAC,SAAS,EAAC,MAAM,OAAO,QAAA,CAAQ,CAAC;AAExE,WAAK,OAAO,KAAK,MAAM,KAAK,YAAY,QAAQ;AAAA,IAClD,OAAO;AACL,WAAK,YAAY,OAAA;AAAA,IACnB;AAAA,EACF;AACF;"}

package/out/zero-cache/src/services/replicator/change-processor.d.ts CHANGED Viewed

@@ -1,10 +1,12 @@
 import type { LogContext } from '@rocicorp/logger';
+import type { DownloadStatus } from '../../../../zero-events/src/status.ts';
 import type { StatementRunner } from '../../db/statements.ts';
 import type { ChangeStreamData } from '../change-source/protocol/current/downstream.ts';
 import type { ReplicatorMode } from './replicator.ts';
 export type ChangeProcessorMode = ReplicatorMode | 'initial-sync';
 export type CommitResult = {
     watermark: string;
+    completedBackfill: DownloadStatus | undefined;
     schemaUpdated: boolean;
     changeLogUpdated: boolean;
 };

package/out/zero-cache/src/services/replicator/change-processor.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-processor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;~~AAuBjD~~,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AAgC5D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,iBAAiB,CAAC;AASpD,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG,cAAc,CAAC;AAElE,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,eAAe;;gBAiBxB,EAAE,EAAE,eAAe,EACnB,IAAI,EAAE,mBAAmB,EACzB,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,KAAK,IAAI;IAuBrD,KAAK,CAAC,EAAE,EAAE,UAAU;IAIpB,8CAA8C;IAC9C,cAAc,CACZ,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,gBAAgB,GAC3B,YAAY,GAAG,IAAI;~~CA6JvB~~"}
1	+ {"version":3,"file":"change-processor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAkB1E,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AAgC5D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,iBAAiB,CAAC;AASpD,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG,cAAc,CAAC;AAElE,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,cAAc,GAAG,SAAS,CAAC;IAC9C,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,eAAe;;gBAiBxB,EAAE,EAAE,eAAe,EACnB,IAAI,EAAE,mBAAmB,EACzB,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,KAAK,IAAI;IAuBrD,KAAK,CAAC,EAAE,EAAE,UAAU;IAIpB,8CAA8C;IAC9C,cAAc,CACZ,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,gBAAgB,GAC3B,YAAY,GAAG,IAAI;CAyJvB"}

package/out/zero-cache/src/services/replicator/change-processor.js CHANGED Viewed

@@ -109,11 +109,7 @@ class ChangeProcessor {
     if (msg.tag === "commit") {
       this.#currentTx = null;
       assert(watermark, "watermark is required for commit messages");
-      const { schemaUpdated, changeLogUpdated } = tx.processCommit(
-        msg,
-        watermark
-      );
-      return { watermark, schemaUpdated, changeLogUpdated };
+      return tx.processCommit(msg, watermark);
     }
     if (msg.tag === "rollback") {
       this.#currentTx?.abort(lc);
@@ -554,7 +550,8 @@ class TransactionProcessor {
       `backfilled ${backfilled} rows (skipped ${skipped}) into ${tableName}`
     );
   }
-  processBackfillCompleted({ relation, columns }) {
+  #completedBackfill;
+  processBackfillCompleted({ relation, columns, status }) {
     const tableName = liteTableName(relation);
     const rowKeyCols = relation.rowKey.columns;
     const cols = [...rowKeyCols, ...columns];
@@ -563,6 +560,9 @@ class TransactionProcessor {
       columnMetadata.clearBackfilling(tableName, col);
     }
     this.#bumpVersions(tableName);
+    if (status) {
+      this.#completedBackfill = { table: tableName, columns: cols, ...status };
+    }
     this.#lc.info?.(`finished backfilling ${tableName}`);
   }
   processCommit(commit, watermark) {
@@ -585,6 +585,8 @@ class TransactionProcessor {
     const elapsedMs = Date.now() - this.#startMs;
     this.#lc.debug?.(`Committed tx@${this.#version} (${elapsedMs} ms)`);
     return {
+      watermark,
+      completedBackfill: this.#completedBackfill,
       schemaUpdated: this.#schemaChanged,
       changeLogUpdated: this.#numChangeLogEntries > 0
     };